From aa99741784e3214eaf15577a0310ab51ba3d4a06 Mon Sep 17 00:00:00 2001 From: amoghjalan Date: Mon, 22 Apr 2024 15:58:31 +0530 Subject: [PATCH 1/2] Restructure Directory structure for backend --- backend/__init__.py | 0 backend/analytics_server/app.py | 34 ++ backend/analytics_server/dora/__init__.py | 0 backend/analytics_server/dora/api/__init__.py | 0 .../dora/api/deployment_analytics.py | 216 ++++++++ backend/analytics_server/dora/api/hello.py | 9 + .../analytics_server/dora/api/incidents.py | 250 +++++++++ .../analytics_server/dora/api/integrations.py | 95 ++++ .../dora/api/pull_requests.py | 169 ++++++ .../dora/api/request_utils.py | 77 +++ .../dora/api/resources/__init__.py | 0 .../dora/api/resources/code_resouces.py | 107 ++++ .../dora/api/resources/core_resources.py | 33 ++ .../api/resources/deployment_resources.py | 38 ++ .../dora/api/resources/incident_resources.py | 71 +++ .../dora/api/resources/settings_resource.py | 61 +++ backend/analytics_server/dora/api/settings.py | 172 ++++++ backend/analytics_server/dora/api/sync.py | 12 + backend/analytics_server/dora/api/teams.py | 79 +++ .../analytics_server/dora/config/config.ini | 3 + .../analytics_server/dora/exapi/__init__.py | 0 .../dora/exapi/git_incidents.py | 74 +++ backend/analytics_server/dora/exapi/github.py | 254 +++++++++ .../dora/exapi/models/__init__.py | 0 .../dora/exapi/models/git_incidents.py | 12 + .../dora/exapi/models/github.py | 45 ++ .../analytics_server/dora/service/__init__.py | 0 .../dora/service/code/__init__.py | 3 + .../dora/service/code/integration.py | 27 + .../dora/service/code/lead_time.py | 264 +++++++++ .../dora/service/code/models/lead_time.py | 49 ++ .../dora/service/code/pr_filter.py | 113 ++++ .../dora/service/code/sync/__init__.py | 1 + .../service/code/sync/etl_code_analytics.py | 173 ++++++ .../service/code/sync/etl_code_factory.py | 13 + .../service/code/sync/etl_github_handler.py | 373 +++++++++++++ .../dora/service/code/sync/etl_handler.py | 125 +++++ .../service/code/sync/etl_provider_handler.py | 53 ++ .../dora/service/code/sync/models.py | 19 + .../code/sync/revert_prs_github_sync.py | 185 +++++++ .../dora/service/core/teams.py | 35 ++ .../dora/service/deployments/__init__.py | 1 + .../dora/service/deployments/analytics.py | 257 +++++++++ .../deployments/deployment_pr_mapper.py | 79 +++ .../service/deployments/deployment_service.py | 171 ++++++ .../deployments_factory_service.py | 46 ++ .../dora/service/deployments/factory.py | 27 + .../service/deployments/models/__init__.py | 0 .../service/deployments/models/adapter.py | 105 ++++ .../dora/service/deployments/models/models.py | 46 ++ .../deployments/pr_deployments_service.py | 51 ++ .../workflow_deployments_service.py | 92 ++++ .../service/external_integrations_service.py | 63 +++ .../dora/service/incidents/__init__.py | 1 + .../dora/service/incidents/incident_filter.py | 109 ++++ .../dora/service/incidents/incidents.py | 213 ++++++++ .../dora/service/incidents/integration.py | 65 +++ .../incidents/models/mean_time_to_recovery.py | 34 ++ .../dora/service/incidents/sync/__init__.py | 1 + .../sync/etl_git_incidents_handler.py | 242 +++++++++ .../service/incidents/sync/etl_handler.py | 107 ++++ .../incidents/sync/etl_incidents_factory.py | 15 + .../incidents/sync/etl_provider_handler.py | 43 ++ .../merge_to_deploy_broker/__init__.py | 2 + .../merge_to_deploy_broker/mtd_handler.py | 126 +++++ .../service/merge_to_deploy_broker/utils.py | 49 ++ .../dora/service/pr_analytics.py | 22 + .../dora/service/query_validator.py | 77 +++ .../dora/service/settings/__init__.py | 2 + .../settings/configuration_settings.py | 376 +++++++++++++ .../service/settings/default_settings_data.py | 29 + .../dora/service/settings/models.py | 41 ++ .../settings/setting_type_validator.py | 19 + .../dora/service/sync_data.py | 34 ++ .../dora/service/workflows/__init__.py | 1 + .../dora/service/workflows/integration.py | 28 + .../dora/service/workflows/sync/__init__.py | 1 + .../sync/etl_github_actions_handler.py | 189 +++++++ .../service/workflows/sync/etl_handler.py | 134 +++++ .../workflows/sync/etl_provider_handler.py | 36 ++ .../workflows/sync/etl_workflows_factory.py | 15 + .../dora/service/workflows/workflow_filter.py | 52 ++ .../analytics_server/dora/store/__init__.py | 36 ++ .../dora/store/initialise_db.py | 27 + .../dora/store/models/__init__.py | 7 + .../dora/store/models/code/__init__.py | 31 ++ .../dora/store/models/code/enums.py | 35 ++ .../dora/store/models/code/filter.py | 98 ++++ .../dora/store/models/code/pull_requests.py | 155 ++++++ .../dora/store/models/code/repository.py | 108 ++++ .../store/models/code/workflows/__init__.py | 3 + .../dora/store/models/code/workflows/enums.py | 32 ++ .../store/models/code/workflows/filter.py | 81 +++ .../store/models/code/workflows/workflows.py | 62 +++ .../dora/store/models/core/__init__.py | 3 + .../dora/store/models/core/organization.py | 23 + .../dora/store/models/core/teams.py | 26 + .../dora/store/models/core/users.py | 21 + .../dora/store/models/incidents/__init__.py | 15 + .../dora/store/models/incidents/enums.py | 35 ++ .../dora/store/models/incidents/filter.py | 45 ++ .../dora/store/models/incidents/incidents.py | 59 ++ .../dora/store/models/incidents/services.py | 45 ++ .../store/models/integrations/__init__.py | 2 + .../dora/store/models/integrations/enums.py | 12 + .../store/models/integrations/integrations.py | 49 ++ .../dora/store/models/settings/__init__.py | 2 + .../models/settings/configuration_settings.py | 33 ++ .../dora/store/models/settings/enums.py | 7 + .../dora/store/repos/__init__.py | 0 .../analytics_server/dora/store/repos/code.py | 333 ++++++++++++ .../analytics_server/dora/store/repos/core.py | 108 ++++ .../dora/store/repos/incidents.py | 148 +++++ .../dora/store/repos/integrations.py | 2 + .../dora/store/repos/settings.py | 90 ++++ .../dora/store/repos/workflows.py | 227 ++++++++ .../analytics_server/dora/utils/__init__.py | 0 .../dora/utils/cryptography.py | 95 ++++ backend/analytics_server/dora/utils/dict.py | 32 ++ backend/analytics_server/dora/utils/github.py | 50 ++ backend/analytics_server/dora/utils/lock.py | 41 ++ backend/analytics_server/dora/utils/log.py | 17 + backend/analytics_server/dora/utils/regex.py | 29 + backend/analytics_server/dora/utils/string.py | 5 + backend/analytics_server/dora/utils/time.py | 270 ++++++++++ .../analytics_server}/env.py | 4 +- backend/analytics_server/tests/__init__.py | 0 .../tests/factories/__init__.py | 0 .../tests/factories/models/__init__.py | 12 + .../tests/factories/models/code.py | 201 +++++++ .../tests/factories/models/exapi/__init__.py | 0 .../tests/factories/models/exapi/github.py | 159 ++++++ .../tests/factories/models/incidents.py | 93 ++++ .../tests/service/Incidents/sync/__init__.py | 0 .../sync/test_etl_git_incidents_handler.py | 141 +++++ .../Incidents/test_change_failure_rate.py | 352 ++++++++++++ .../test_deployment_incident_mapper.py | 119 +++++ .../tests/service/__init__.py | 0 .../tests/service/code/__init__.py | 0 .../tests/service/code/sync/__init__.py | 0 .../code/sync/test_etl_code_analytics.py | 505 ++++++++++++++++++ .../code/sync/test_etl_github_handler.py | 353 ++++++++++++ .../service/code/test_lead_time_service.py | 198 +++++++ .../tests/service/deployments/__init__.py | 0 .../deployments/test_deployment_frequency.py | 181 +++++++ .../deployments/test_deployment_pr_mapper.py | 183 +++++++ .../tests/service/workflows/__init__.py | 0 .../tests/service/workflows/sync/__init__.py | 0 .../sync/test_etl_github_actions_handler.py | 110 ++++ backend/analytics_server/tests/utilities.py | 20 + .../dict/test_get_average_of_dict_values.py | 17 + .../utils/dict/test_get_key_to_count_map.py | 29 + .../time/test_fill_missing_week_buckets.py | 74 +++ .../time/test_generate_expanded_buckets.py | 288 ++++++++++ {apiserver => backend}/dev-requirements.txt | 0 {apiserver => backend}/env.example | 0 {apiserver => backend}/requirements.txt | 0 157 files changed, 11781 insertions(+), 2 deletions(-) create mode 100644 backend/__init__.py create mode 100644 backend/analytics_server/app.py create mode 100644 backend/analytics_server/dora/__init__.py create mode 100644 backend/analytics_server/dora/api/__init__.py create mode 100644 backend/analytics_server/dora/api/deployment_analytics.py create mode 100644 backend/analytics_server/dora/api/hello.py create mode 100644 backend/analytics_server/dora/api/incidents.py create mode 100644 backend/analytics_server/dora/api/integrations.py create mode 100644 backend/analytics_server/dora/api/pull_requests.py create mode 100644 backend/analytics_server/dora/api/request_utils.py create mode 100644 backend/analytics_server/dora/api/resources/__init__.py create mode 100644 backend/analytics_server/dora/api/resources/code_resouces.py create mode 100644 backend/analytics_server/dora/api/resources/core_resources.py create mode 100644 backend/analytics_server/dora/api/resources/deployment_resources.py create mode 100644 backend/analytics_server/dora/api/resources/incident_resources.py create mode 100644 backend/analytics_server/dora/api/resources/settings_resource.py create mode 100644 backend/analytics_server/dora/api/settings.py create mode 100644 backend/analytics_server/dora/api/sync.py create mode 100644 backend/analytics_server/dora/api/teams.py create mode 100644 backend/analytics_server/dora/config/config.ini create mode 100644 backend/analytics_server/dora/exapi/__init__.py create mode 100644 backend/analytics_server/dora/exapi/git_incidents.py create mode 100644 backend/analytics_server/dora/exapi/github.py create mode 100644 backend/analytics_server/dora/exapi/models/__init__.py create mode 100644 backend/analytics_server/dora/exapi/models/git_incidents.py create mode 100644 backend/analytics_server/dora/exapi/models/github.py create mode 100644 backend/analytics_server/dora/service/__init__.py create mode 100644 backend/analytics_server/dora/service/code/__init__.py create mode 100644 backend/analytics_server/dora/service/code/integration.py create mode 100644 backend/analytics_server/dora/service/code/lead_time.py create mode 100644 backend/analytics_server/dora/service/code/models/lead_time.py create mode 100644 backend/analytics_server/dora/service/code/pr_filter.py create mode 100644 backend/analytics_server/dora/service/code/sync/__init__.py create mode 100644 backend/analytics_server/dora/service/code/sync/etl_code_analytics.py create mode 100644 backend/analytics_server/dora/service/code/sync/etl_code_factory.py create mode 100644 backend/analytics_server/dora/service/code/sync/etl_github_handler.py create mode 100644 backend/analytics_server/dora/service/code/sync/etl_handler.py create mode 100644 backend/analytics_server/dora/service/code/sync/etl_provider_handler.py create mode 100644 backend/analytics_server/dora/service/code/sync/models.py create mode 100644 backend/analytics_server/dora/service/code/sync/revert_prs_github_sync.py create mode 100644 backend/analytics_server/dora/service/core/teams.py create mode 100644 backend/analytics_server/dora/service/deployments/__init__.py create mode 100644 backend/analytics_server/dora/service/deployments/analytics.py create mode 100644 backend/analytics_server/dora/service/deployments/deployment_pr_mapper.py create mode 100644 backend/analytics_server/dora/service/deployments/deployment_service.py create mode 100644 backend/analytics_server/dora/service/deployments/deployments_factory_service.py create mode 100644 backend/analytics_server/dora/service/deployments/factory.py create mode 100644 backend/analytics_server/dora/service/deployments/models/__init__.py create mode 100644 backend/analytics_server/dora/service/deployments/models/adapter.py create mode 100644 backend/analytics_server/dora/service/deployments/models/models.py create mode 100644 backend/analytics_server/dora/service/deployments/pr_deployments_service.py create mode 100644 backend/analytics_server/dora/service/deployments/workflow_deployments_service.py create mode 100644 backend/analytics_server/dora/service/external_integrations_service.py create mode 100644 backend/analytics_server/dora/service/incidents/__init__.py create mode 100644 backend/analytics_server/dora/service/incidents/incident_filter.py create mode 100644 backend/analytics_server/dora/service/incidents/incidents.py create mode 100644 backend/analytics_server/dora/service/incidents/integration.py create mode 100644 backend/analytics_server/dora/service/incidents/models/mean_time_to_recovery.py create mode 100644 backend/analytics_server/dora/service/incidents/sync/__init__.py create mode 100644 backend/analytics_server/dora/service/incidents/sync/etl_git_incidents_handler.py create mode 100644 backend/analytics_server/dora/service/incidents/sync/etl_handler.py create mode 100644 backend/analytics_server/dora/service/incidents/sync/etl_incidents_factory.py create mode 100644 backend/analytics_server/dora/service/incidents/sync/etl_provider_handler.py create mode 100644 backend/analytics_server/dora/service/merge_to_deploy_broker/__init__.py create mode 100644 backend/analytics_server/dora/service/merge_to_deploy_broker/mtd_handler.py create mode 100644 backend/analytics_server/dora/service/merge_to_deploy_broker/utils.py create mode 100644 backend/analytics_server/dora/service/pr_analytics.py create mode 100644 backend/analytics_server/dora/service/query_validator.py create mode 100644 backend/analytics_server/dora/service/settings/__init__.py create mode 100644 backend/analytics_server/dora/service/settings/configuration_settings.py create mode 100644 backend/analytics_server/dora/service/settings/default_settings_data.py create mode 100644 backend/analytics_server/dora/service/settings/models.py create mode 100644 backend/analytics_server/dora/service/settings/setting_type_validator.py create mode 100644 backend/analytics_server/dora/service/sync_data.py create mode 100644 backend/analytics_server/dora/service/workflows/__init__.py create mode 100644 backend/analytics_server/dora/service/workflows/integration.py create mode 100644 backend/analytics_server/dora/service/workflows/sync/__init__.py create mode 100644 backend/analytics_server/dora/service/workflows/sync/etl_github_actions_handler.py create mode 100644 backend/analytics_server/dora/service/workflows/sync/etl_handler.py create mode 100644 backend/analytics_server/dora/service/workflows/sync/etl_provider_handler.py create mode 100644 backend/analytics_server/dora/service/workflows/sync/etl_workflows_factory.py create mode 100644 backend/analytics_server/dora/service/workflows/workflow_filter.py create mode 100644 backend/analytics_server/dora/store/__init__.py create mode 100644 backend/analytics_server/dora/store/initialise_db.py create mode 100644 backend/analytics_server/dora/store/models/__init__.py create mode 100644 backend/analytics_server/dora/store/models/code/__init__.py create mode 100644 backend/analytics_server/dora/store/models/code/enums.py create mode 100644 backend/analytics_server/dora/store/models/code/filter.py create mode 100644 backend/analytics_server/dora/store/models/code/pull_requests.py create mode 100644 backend/analytics_server/dora/store/models/code/repository.py create mode 100644 backend/analytics_server/dora/store/models/code/workflows/__init__.py create mode 100644 backend/analytics_server/dora/store/models/code/workflows/enums.py create mode 100644 backend/analytics_server/dora/store/models/code/workflows/filter.py create mode 100644 backend/analytics_server/dora/store/models/code/workflows/workflows.py create mode 100644 backend/analytics_server/dora/store/models/core/__init__.py create mode 100644 backend/analytics_server/dora/store/models/core/organization.py create mode 100644 backend/analytics_server/dora/store/models/core/teams.py create mode 100644 backend/analytics_server/dora/store/models/core/users.py create mode 100644 backend/analytics_server/dora/store/models/incidents/__init__.py create mode 100644 backend/analytics_server/dora/store/models/incidents/enums.py create mode 100644 backend/analytics_server/dora/store/models/incidents/filter.py create mode 100644 backend/analytics_server/dora/store/models/incidents/incidents.py create mode 100644 backend/analytics_server/dora/store/models/incidents/services.py create mode 100644 backend/analytics_server/dora/store/models/integrations/__init__.py create mode 100644 backend/analytics_server/dora/store/models/integrations/enums.py create mode 100644 backend/analytics_server/dora/store/models/integrations/integrations.py create mode 100644 backend/analytics_server/dora/store/models/settings/__init__.py create mode 100644 backend/analytics_server/dora/store/models/settings/configuration_settings.py create mode 100644 backend/analytics_server/dora/store/models/settings/enums.py create mode 100644 backend/analytics_server/dora/store/repos/__init__.py create mode 100644 backend/analytics_server/dora/store/repos/code.py create mode 100644 backend/analytics_server/dora/store/repos/core.py create mode 100644 backend/analytics_server/dora/store/repos/incidents.py create mode 100644 backend/analytics_server/dora/store/repos/integrations.py create mode 100644 backend/analytics_server/dora/store/repos/settings.py create mode 100644 backend/analytics_server/dora/store/repos/workflows.py create mode 100644 backend/analytics_server/dora/utils/__init__.py create mode 100644 backend/analytics_server/dora/utils/cryptography.py create mode 100644 backend/analytics_server/dora/utils/dict.py create mode 100644 backend/analytics_server/dora/utils/github.py create mode 100644 backend/analytics_server/dora/utils/lock.py create mode 100644 backend/analytics_server/dora/utils/log.py create mode 100644 backend/analytics_server/dora/utils/regex.py create mode 100644 backend/analytics_server/dora/utils/string.py create mode 100644 backend/analytics_server/dora/utils/time.py rename {apiserver => backend/analytics_server}/env.py (64%) create mode 100644 backend/analytics_server/tests/__init__.py create mode 100644 backend/analytics_server/tests/factories/__init__.py create mode 100644 backend/analytics_server/tests/factories/models/__init__.py create mode 100644 backend/analytics_server/tests/factories/models/code.py create mode 100644 backend/analytics_server/tests/factories/models/exapi/__init__.py create mode 100644 backend/analytics_server/tests/factories/models/exapi/github.py create mode 100644 backend/analytics_server/tests/factories/models/incidents.py create mode 100644 backend/analytics_server/tests/service/Incidents/sync/__init__.py create mode 100644 backend/analytics_server/tests/service/Incidents/sync/test_etl_git_incidents_handler.py create mode 100644 backend/analytics_server/tests/service/Incidents/test_change_failure_rate.py create mode 100644 backend/analytics_server/tests/service/Incidents/test_deployment_incident_mapper.py create mode 100644 backend/analytics_server/tests/service/__init__.py create mode 100644 backend/analytics_server/tests/service/code/__init__.py create mode 100644 backend/analytics_server/tests/service/code/sync/__init__.py create mode 100644 backend/analytics_server/tests/service/code/sync/test_etl_code_analytics.py create mode 100644 backend/analytics_server/tests/service/code/sync/test_etl_github_handler.py create mode 100644 backend/analytics_server/tests/service/code/test_lead_time_service.py create mode 100644 backend/analytics_server/tests/service/deployments/__init__.py create mode 100644 backend/analytics_server/tests/service/deployments/test_deployment_frequency.py create mode 100644 backend/analytics_server/tests/service/deployments/test_deployment_pr_mapper.py create mode 100644 backend/analytics_server/tests/service/workflows/__init__.py create mode 100644 backend/analytics_server/tests/service/workflows/sync/__init__.py create mode 100644 backend/analytics_server/tests/service/workflows/sync/test_etl_github_actions_handler.py create mode 100644 backend/analytics_server/tests/utilities.py create mode 100644 backend/analytics_server/tests/utils/dict/test_get_average_of_dict_values.py create mode 100644 backend/analytics_server/tests/utils/dict/test_get_key_to_count_map.py create mode 100644 backend/analytics_server/tests/utils/time/test_fill_missing_week_buckets.py create mode 100644 backend/analytics_server/tests/utils/time/test_generate_expanded_buckets.py rename {apiserver => backend}/dev-requirements.txt (100%) rename {apiserver => backend}/env.example (100%) rename {apiserver => backend}/requirements.txt (100%) diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/app.py b/backend/analytics_server/app.py new file mode 100644 index 000000000..e310eee9f --- /dev/null +++ b/backend/analytics_server/app.py @@ -0,0 +1,34 @@ +from flask import Flask + +from dora.store import configure_db_with_app +from env import load_app_env + +load_app_env() + +from dora.api.hello import app as core_api +from dora.api.settings import app as settings_api +from dora.api.pull_requests import app as pull_requests_api +from dora.api.incidents import app as incidents_api +from dora.api.integrations import app as integrations_api +from dora.api.deployment_analytics import app as deployment_analytics_api +from dora.api.teams import app as teams_api +from dora.api.sync import app as sync_api + +from dora.store.initialise_db import initialize_database + +app = Flask(__name__) + +app.register_blueprint(core_api) +app.register_blueprint(settings_api) +app.register_blueprint(pull_requests_api) +app.register_blueprint(incidents_api) +app.register_blueprint(deployment_analytics_api) +app.register_blueprint(integrations_api) +app.register_blueprint(teams_api) +app.register_blueprint(sync_api) + +configure_db_with_app(app) +initialize_database(app) + +if __name__ == "__main__": + app.run() diff --git a/backend/analytics_server/dora/__init__.py b/backend/analytics_server/dora/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/dora/api/__init__.py b/backend/analytics_server/dora/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/dora/api/deployment_analytics.py b/backend/analytics_server/dora/api/deployment_analytics.py new file mode 100644 index 000000000..f6d82461b --- /dev/null +++ b/backend/analytics_server/dora/api/deployment_analytics.py @@ -0,0 +1,216 @@ +from werkzeug.exceptions import NotFound +from collections import defaultdict +from typing import Dict, List +from datetime import datetime +import json + +from flask import Blueprint +from voluptuous import Required, Schema, Coerce, All, Optional +from dora.api.resources.code_resouces import get_non_paginated_pr_response +from dora.service.deployments.deployments_factory_service import ( + DeploymentsFactoryService, +) +from dora.service.deployments.factory import get_deployments_factory +from dora.service.pr_analytics import get_pr_analytics_service +from dora.service.code.pr_filter import apply_pr_filter + +from dora.api.request_utils import coerce_workflow_filter, queryschema +from dora.api.resources.deployment_resources import ( + adapt_deployment, + adapt_deployment_frequency_metrics, +) +from dora.service.deployments.analytics import get_deployment_analytics_service +from dora.service.query_validator import get_query_validator +from dora.store.models import SettingType, EntityType, Team +from dora.store.models.code.filter import PRFilter +from dora.store.models.code.pull_requests import PullRequest +from dora.store.models.code.repository import OrgRepo, TeamRepos +from dora.store.models.code.workflows.filter import WorkflowFilter +from dora.service.deployments.models.models import ( + Deployment, + DeploymentFrequencyMetrics, + DeploymentType, +) +from dora.store.repos.code import CodeRepoService + + +app = Blueprint("deployment_analytics", __name__) + + +@app.route("/teams//deployment_analytics", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), + Optional("workflow_filter"): All(str, Coerce(coerce_workflow_filter)), + } + ), +) +def get_team_deployment_analytics( + team_id: str, + from_time: datetime, + to_time: datetime, + pr_filter: Dict = None, + workflow_filter: WorkflowFilter = None, +): + query_validator = get_query_validator() + interval = query_validator.interval_validator(from_time, to_time) + query_validator.team_validator(team_id) + + pr_filter: PRFilter = apply_pr_filter( + pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] + ) + code_repo_service = CodeRepoService() + + team_repos: List[TeamRepos] = code_repo_service.get_active_team_repos_by_team_id( + team_id + ) + org_repos: List[OrgRepo] = code_repo_service.get_active_org_repos_by_ids( + [str(team_repo.org_repo_id) for team_repo in team_repos] + ) + + deployments_analytics_service = get_deployment_analytics_service() + + repo_id_to_deployments_map_with_prs: Dict[ + str, List[Dict[Deployment, List[PullRequest]]] + ] = deployments_analytics_service.get_team_successful_deployments_in_interval_with_related_prs( + team_id, interval, pr_filter, workflow_filter + ) + + repo_id_deployments_map = defaultdict(list) + + for repo_id, deployment_to_prs_map in repo_id_to_deployments_map_with_prs.items(): + adapted_deployments = [] + for deployment, prs in deployment_to_prs_map.items(): + adapted_deployment = adapt_deployment(deployment) + adapted_deployment["pr_count"] = len(prs) + + adapted_deployments.append(adapted_deployment) + + repo_id_deployments_map[repo_id] = adapted_deployments + + return { + "deployments_map": repo_id_deployments_map, + "repos_map": { + str(repo.id): { + "id": str(repo.id), + "name": repo.name, + "language": repo.language, + "default_branch": repo.default_branch, + "parent": repo.org_name, + } + for repo in org_repos + }, + } + + +@app.route("/deployments//prs", methods={"GET"}) +def get_prs_included_in_deployment(deployment_id: str): + pr_analytics_service = get_pr_analytics_service() + deployment_type: DeploymentType + + ( + deployment_type, + entity_id, + ) = DeploymentsFactoryService.get_deployment_type_and_entity_id_from_deployment_id( + deployment_id + ) + + deployments_service: DeploymentsFactoryService = get_deployments_factory( + deployment_type + ) + deployment: Deployment = deployments_service.get_deployment_by_entity_id(entity_id) + if not deployment: + raise NotFound(f"Deployment not found for id {deployment_id}") + + repo: OrgRepo = pr_analytics_service.get_repo_by_id(deployment.repo_id) + + prs: List[ + PullRequest + ] = deployments_service.get_pull_requests_related_to_deployment(deployment) + repo_id_map = {repo.id: repo} + + return get_non_paginated_pr_response( + prs=prs, repo_id_map=repo_id_map, total_count=len(prs) + ) + + +@app.route("/teams//deployment_frequency", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), + Optional("workflow_filter"): All(str, Coerce(coerce_workflow_filter)), + } + ), +) +def get_team_deployment_frequency( + team_id: str, + from_time: datetime, + to_time: datetime, + pr_filter: Dict = None, + workflow_filter: WorkflowFilter = None, +): + + query_validator = get_query_validator() + interval = query_validator.interval_validator(from_time, to_time) + query_validator.team_validator(team_id) + + pr_filter: PRFilter = apply_pr_filter( + pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] + ) + + deployments_analytics_service = get_deployment_analytics_service() + + team_deployment_frequency_metrics: DeploymentFrequencyMetrics = ( + deployments_analytics_service.get_team_deployment_frequency_metrics( + team_id, interval, pr_filter, workflow_filter + ) + ) + + return adapt_deployment_frequency_metrics(team_deployment_frequency_metrics) + + +@app.route("/teams//deployment_frequency/trends", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), + Optional("workflow_filter"): All(str, Coerce(coerce_workflow_filter)), + } + ), +) +def get_team_deployment_frequency_trends( + team_id: str, + from_time: datetime, + to_time: datetime, + pr_filter: Dict = None, + workflow_filter: WorkflowFilter = None, +): + + query_validator = get_query_validator() + interval = query_validator.interval_validator(from_time, to_time) + query_validator.team_validator(team_id) + + pr_filter: PRFilter = apply_pr_filter( + pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] + ) + + deployments_analytics_service = get_deployment_analytics_service() + + week_to_deployments_count_map: Dict[ + datetime, int + ] = deployments_analytics_service.get_weekly_deployment_frequency_trends( + team_id, interval, pr_filter, workflow_filter + ) + + return { + week.isoformat(): {"count": deployment_count} + for week, deployment_count in week_to_deployments_count_map.items() + } diff --git a/backend/analytics_server/dora/api/hello.py b/backend/analytics_server/dora/api/hello.py new file mode 100644 index 000000000..11504aee9 --- /dev/null +++ b/backend/analytics_server/dora/api/hello.py @@ -0,0 +1,9 @@ +from flask import Blueprint + +app = Blueprint("hello", __name__) + + +@app.route("/", methods=["GET"]) +def hello_world(): + + return {"message": "hello world"} diff --git a/backend/analytics_server/dora/api/incidents.py b/backend/analytics_server/dora/api/incidents.py new file mode 100644 index 000000000..4852b69f8 --- /dev/null +++ b/backend/analytics_server/dora/api/incidents.py @@ -0,0 +1,250 @@ +import json +from typing import Dict, List + +from datetime import datetime + +from flask import Blueprint +from voluptuous import Required, Schema, Coerce, All, Optional +from dora.service.code.pr_filter import apply_pr_filter +from dora.store.models.code.filter import PRFilter +from dora.store.models.settings import SettingType, EntityType +from dora.service.incidents.models.mean_time_to_recovery import ChangeFailureRateMetrics +from dora.service.deployments.deployment_service import ( + get_deployments_service, +) +from dora.service.deployments.models.models import Deployment +from dora.store.models.code.workflows.filter import WorkflowFilter +from dora.utils.time import Interval +from dora.service.incidents.incidents import get_incident_service +from dora.api.resources.incident_resources import ( + adapt_change_failure_rate, + adapt_deployments_with_related_incidents, + adapt_incident, + adapt_mean_time_to_recovery_metrics, +) +from dora.store.models.incidents import Incident + +from dora.api.request_utils import coerce_workflow_filter, queryschema +from dora.service.query_validator import get_query_validator + +app = Blueprint("incidents", __name__) + + +@app.route("/teams//resolved_incidents", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + } + ), +) +def get_resolved_incidents(team_id: str, from_time: datetime, to_time: datetime): + + query_validator = get_query_validator() + interval = query_validator.interval_validator(from_time, to_time) + query_validator.team_validator(team_id) + + incident_service = get_incident_service() + + resolved_incidents: List[Incident] = incident_service.get_resolved_team_incidents( + team_id, interval + ) + + # ToDo: Generate a user map + + return [adapt_incident(incident) for incident in resolved_incidents] + + +@app.route("/teams//deployments_with_related_incidents", methods=["GET"]) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), + Optional("workflow_filter"): All(str, Coerce(coerce_workflow_filter)), + } + ), +) +def get_deployments_with_related_incidents( + team_id: str, + from_time: datetime, + to_time: datetime, + pr_filter: dict = None, + workflow_filter: WorkflowFilter = None, +): + query_validator = get_query_validator() + interval = Interval(from_time, to_time) + query_validator.team_validator(team_id) + + pr_filter: PRFilter = apply_pr_filter( + pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] + ) + + deployments: List[ + Deployment + ] = get_deployments_service().get_team_all_deployments_in_interval( + team_id, interval, pr_filter, workflow_filter + ) + + incident_service = get_incident_service() + + incidents: List[Incident] = incident_service.get_team_incidents(team_id, interval) + + deployment_incidents_map: Dict[ + Deployment, List[Incident] + ] = incident_service.get_deployment_incidents_map(deployments, incidents) + + return list( + map( + lambda deployment: adapt_deployments_with_related_incidents( + deployment, deployment_incidents_map + ), + deployments, + ) + ) + + +@app.route("/teams//mean_time_to_recovery", methods=["GET"]) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + } + ), +) +def get_team_mttr(team_id: str, from_time: datetime, to_time: datetime): + query_validator = get_query_validator() + interval = query_validator.interval_validator(from_time, to_time) + query_validator.team_validator(team_id) + + incident_service = get_incident_service() + + team_mean_time_to_recovery_metrics = ( + incident_service.get_team_mean_time_to_recovery(team_id, interval) + ) + + return adapt_mean_time_to_recovery_metrics(team_mean_time_to_recovery_metrics) + + +@app.route("/teams//mean_time_to_recovery/trends", methods=["GET"]) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + } + ), +) +def get_team_mttr_trends(team_id: str, from_time: datetime, to_time: datetime): + query_validator = get_query_validator() + interval = query_validator.interval_validator(from_time, to_time) + query_validator.team_validator(team_id) + + incident_service = get_incident_service() + + weekly_mean_time_to_recovery_metrics = ( + incident_service.get_team_mean_time_to_recovery_trends(team_id, interval) + ) + + return { + week.isoformat(): adapt_mean_time_to_recovery_metrics( + mean_time_to_recovery_metrics + ) + for week, mean_time_to_recovery_metrics in weekly_mean_time_to_recovery_metrics.items() + } + + +@app.route("/teams//change_failure_rate", methods=["GET"]) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), + Optional("workflow_filter"): All(str, Coerce(coerce_workflow_filter)), + } + ), +) +def get_team_cfr( + team_id: str, + from_time: datetime, + to_time: datetime, + pr_filter: dict = None, + workflow_filter: WorkflowFilter = None, +): + + query_validator = get_query_validator() + interval = Interval(from_time, to_time) + query_validator.team_validator(team_id) + + pr_filter: PRFilter = apply_pr_filter( + pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] + ) + + deployments: List[ + Deployment + ] = get_deployments_service().get_team_all_deployments_in_interval( + team_id, interval, pr_filter, workflow_filter + ) + + incident_service = get_incident_service() + + incidents: List[Incident] = incident_service.get_team_incidents(team_id, interval) + + team_change_failure_rate: ChangeFailureRateMetrics = ( + incident_service.get_change_failure_rate_metrics(deployments, incidents) + ) + + return adapt_change_failure_rate(team_change_failure_rate) + + +@app.route("/teams//change_failure_rate/trends", methods=["GET"]) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), + Optional("workflow_filter"): All(str, Coerce(coerce_workflow_filter)), + } + ), +) +def get_team_cfr_trends( + team_id: str, + from_time: datetime, + to_time: datetime, + pr_filter: dict = None, + workflow_filter: WorkflowFilter = None, +): + + query_validator = get_query_validator() + interval = Interval(from_time, to_time) + query_validator.team_validator(team_id) + + pr_filter: PRFilter = apply_pr_filter( + pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] + ) + + deployments: List[ + Deployment + ] = get_deployments_service().get_team_all_deployments_in_interval( + team_id, interval, pr_filter, workflow_filter + ) + + incident_service = get_incident_service() + + incidents: List[Incident] = incident_service.get_team_incidents(team_id, interval) + + team_weekly_change_failure_rate: Dict[ + datetime, ChangeFailureRateMetrics + ] = incident_service.get_weekly_change_failure_rate( + interval, deployments, incidents + ) + + return { + week.isoformat(): adapt_change_failure_rate(change_failure_rate) + for week, change_failure_rate in team_weekly_change_failure_rate.items() + } diff --git a/backend/analytics_server/dora/api/integrations.py b/backend/analytics_server/dora/api/integrations.py new file mode 100644 index 000000000..1d6a7db08 --- /dev/null +++ b/backend/analytics_server/dora/api/integrations.py @@ -0,0 +1,95 @@ +from flask import Blueprint +from voluptuous import Schema, Optional, Coerce, Range, All, Required + +from dora.api.request_utils import queryschema +from dora.service.external_integrations_service import get_external_integrations_service +from dora.service.query_validator import get_query_validator +from dora.store.models import UserIdentityProvider +from dora.utils.github import github_org_data_multi_thread_worker + +app = Blueprint("integrations", __name__) + +STATUS_TOO_MANY_REQUESTS = 429 + + +@app.route("/orgs//integrations/github/orgs", methods={"GET"}) +def get_github_orgs(org_id: str): + query_validator = get_query_validator() + query_validator.org_validator(org_id) + + external_integrations_service = get_external_integrations_service( + org_id, UserIdentityProvider.GITHUB + ) + try: + orgs = external_integrations_service.get_github_organizations() + except Exception as e: + return e, STATUS_TOO_MANY_REQUESTS + org_data_map = github_org_data_multi_thread_worker(orgs) + + return { + "orgs": [ + { + "login": o.login, + "avatar_url": o.avatar_url, + "web_url": o.html_url, + "repos": org_data_map.get(o.name, {}).get("repos", []), + "members": [], + } + for o in orgs + ] + } + + +@app.route("/orgs//integrations/github/orgs//repos", methods={"GET"}) +@queryschema( + Schema( + { + Optional("page_size", default="30"): All( + str, Coerce(int), Range(min=1, max=100) + ), + Optional("page", default="1"): All(str, Coerce(int), Range(min=1)), + } + ), +) +def get_repos(org_id: str, org_login: str, page_size: int, page: int): + query_validator = get_query_validator() + query_validator.org_validator(org_id) + + external_integrations_service = get_external_integrations_service( + org_id, UserIdentityProvider.GITHUB + ) + # GitHub pages start from 0 and Bitbucket pages start from 1. + # Need to be consistent, hence making standard as page starting from 1 + # and passing a decremented value to GitHub + try: + return external_integrations_service.get_github_org_repos( + org_login, page_size, page - 1 + ) + except Exception as e: + return e, STATUS_TOO_MANY_REQUESTS + + +@app.route( + "/orgs//integrations/github///workflows", + methods={"GET"}, +) +def get_prs_for_repo(org_id: str, gh_org_name: str, gh_org_repo_name: str): + query_validator = get_query_validator() + query_validator.org_validator(org_id) + + external_integrations_service = get_external_integrations_service( + org_id, UserIdentityProvider.GITHUB + ) + + workflows_list = external_integrations_service.get_repo_workflows( + gh_org_name, gh_org_repo_name + ) + + return [ + { + "id": github_workflow.id, + "name": github_workflow.name, + "html_url": github_workflow.html_url, + } + for github_workflow in workflows_list + ] diff --git a/backend/analytics_server/dora/api/pull_requests.py b/backend/analytics_server/dora/api/pull_requests.py new file mode 100644 index 000000000..d1b664818 --- /dev/null +++ b/backend/analytics_server/dora/api/pull_requests.py @@ -0,0 +1,169 @@ +import json +from datetime import datetime + +from flask import Blueprint +from typing import Dict, List + +from voluptuous import Required, Schema, Coerce, All, Optional +from dora.service.code.models.lead_time import LeadTimeMetrics +from dora.service.code.lead_time import get_lead_time_service +from dora.service.code.pr_filter import apply_pr_filter + +from dora.store.models.code import PRFilter +from dora.store.models.core import Team +from dora.service.query_validator import get_query_validator + +from dora.api.request_utils import queryschema +from dora.api.resources.code_resouces import ( + adapt_lead_time_metrics, + adapt_pull_request, + get_non_paginated_pr_response, +) +from dora.store.models.code.pull_requests import PullRequest +from dora.service.pr_analytics import get_pr_analytics_service +from dora.service.settings.models import ExcludedPRsSetting + +from dora.utils.time import Interval + + +from dora.service.settings.configuration_settings import get_settings_service + +from dora.store.models import SettingType, EntityType + + +app = Blueprint("pull_requests", __name__) + + +@app.route("/teams//prs/excluded", methods={"GET"}) +def get_team_excluded_prs(team_id: str): + + settings = get_settings_service().get_settings( + setting_type=SettingType.EXCLUDED_PRS_SETTING, + entity_id=team_id, + entity_type=EntityType.TEAM, + ) + + if not settings: + return [] + + excluded_pr_setting: ExcludedPRsSetting = settings.specific_settings + + excluded_pr_ids = excluded_pr_setting.excluded_pr_ids + + pr_analytics_service = get_pr_analytics_service() + + prs: List[PullRequest] = pr_analytics_service.get_prs_by_ids(excluded_pr_ids) + + return [adapt_pull_request(pr) for pr in prs] + + +@app.route("/teams//lead_time/prs", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), + } + ), +) +def get_lead_time_prs( + team_id: str, + from_time: datetime, + to_time: datetime, + pr_filter: Dict = None, +): + + query_validator = get_query_validator() + + interval: Interval = query_validator.interval_validator(from_time, to_time) + team: Team = query_validator.team_validator(team_id) + + pr_filter: PRFilter = apply_pr_filter( + pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] + ) + + lead_time_service = get_lead_time_service() + pr_analytics = get_pr_analytics_service() + + repos = pr_analytics.get_team_repos(team_id) + + prs = lead_time_service.get_team_lead_time_prs(team, interval, pr_filter) + + repo_id_repo_map = {repo.id: repo for repo in repos} + return get_non_paginated_pr_response(prs, repo_id_repo_map, len(prs)) + + +@app.route("/teams//lead_time", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), + } + ), +) +def get_team_lead_time( + team_id: str, + from_time: datetime, + to_time: datetime, + pr_filter: Dict = None, +): + + query_validator = get_query_validator() + + interval: Interval = query_validator.interval_validator(from_time, to_time) + team: Team = query_validator.team_validator(team_id) + + pr_filter: PRFilter = apply_pr_filter( + pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] + ) + + lead_time_service = get_lead_time_service() + + teams_average_lead_time_metrics = lead_time_service.get_team_lead_time_metrics( + team, interval, pr_filter + ) + + adapted_lead_time_metrics = adapt_lead_time_metrics(teams_average_lead_time_metrics) + + return adapted_lead_time_metrics + + +@app.route("/teams//lead_time/trends", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), + } + ), +) +def get_team_lead_time_trends( + team_id: str, + from_time: datetime, + to_time: datetime, + pr_filter: Dict = None, +): + + query_validator = get_query_validator() + + interval: Interval = query_validator.interval_validator(from_time, to_time) + team: Team = query_validator.team_validator(team_id) + + pr_filter: PRFilter = apply_pr_filter( + pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] + ) + + lead_time_service = get_lead_time_service() + + weekly_lead_time_metrics_avg_map: Dict[ + datetime, LeadTimeMetrics + ] = lead_time_service.get_team_lead_time_metrics_trends(team, interval, pr_filter) + + return { + week.isoformat(): adapt_lead_time_metrics(average_lead_time_metrics) + for week, average_lead_time_metrics in weekly_lead_time_metrics_avg_map.items() + } diff --git a/backend/analytics_server/dora/api/request_utils.py b/backend/analytics_server/dora/api/request_utils.py new file mode 100644 index 000000000..7ccb843bd --- /dev/null +++ b/backend/analytics_server/dora/api/request_utils.py @@ -0,0 +1,77 @@ +from functools import wraps +from uuid import UUID + +from flask import request +from stringcase import snakecase +from voluptuous import Invalid +from werkzeug.exceptions import BadRequest +from dora.store.models.code.workflows import WorkflowFilter + +from dora.service.workflows.workflow_filter import get_workflow_filter_processor + + +def queryschema(schema): + def decorator(f): + @wraps(f) + def new_func(*args, **kwargs): + try: + query_params = request.args.to_dict() + valid_dict = schema(dict(query_params)) + snaked_kwargs = {snakecase(k): v for k, v in valid_dict.items()} + kwargs.update(snaked_kwargs) + except Invalid as e: + message = "Invalid data: %s (path %s)" % ( + str(e.msg), + ".".join([str(k) for k in e.path]), + ) + raise BadRequest(message) + + return f(*args, **kwargs) + + return new_func + + return decorator + + +def uuid_validator(s: str): + UUID(s) + return s + + +def boolean_validator(s: str): + if s.lower() == "true" or s == "1": + return True + elif s.lower() == "false" or s == "0": + return False + else: + raise ValueError("Not a boolean") + + +def dataschema(schema): + def decorator(f): + @wraps(f) + def new_func(*args, **kwargs): + try: + body = request.json or {} + valid_dict = schema(body) + snaked_kwargs = {snakecase(k): v for k, v in valid_dict.items()} + kwargs.update(snaked_kwargs) + except Invalid as e: + message = "Invalid data: %s (path %s)" % ( + str(e.msg), + ".".join([str(k) for k in e.path]), + ) + raise BadRequest(message) + + return f(*args, **kwargs) + + return new_func + + return decorator + + +def coerce_workflow_filter(filter_data: str) -> WorkflowFilter: + workflow_filter_processor = get_workflow_filter_processor() + return workflow_filter_processor.create_workflow_filter_from_json_string( + filter_data + ) diff --git a/backend/analytics_server/dora/api/resources/__init__.py b/backend/analytics_server/dora/api/resources/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/dora/api/resources/code_resouces.py b/backend/analytics_server/dora/api/resources/code_resouces.py new file mode 100644 index 000000000..9967c6e72 --- /dev/null +++ b/backend/analytics_server/dora/api/resources/code_resouces.py @@ -0,0 +1,107 @@ +from typing import Dict, List +from dora.service.code.models.lead_time import LeadTimeMetrics +from dora.api.resources.core_resources import adapt_user_info +from dora.store.models.code import PullRequest +from dora.store.models.core import Users + + +def adapt_pull_request( + pr: PullRequest, + username_user_map: Dict[str, Users] = None, +) -> Dict[str, any]: + username_user_map = username_user_map or {} + pr_data = { + "id": str(pr.id), + "repo_id": str(pr.repo_id), + "number": pr.number, + "title": pr.title, + "state": pr.state.value, + "author": adapt_user_info(pr.author, username_user_map), + "reviewers": [ + adapt_user_info(r, username_user_map) for r in (pr.reviewers or []) + ], + "url": pr.url, + "base_branch": pr.base_branch, + "head_branch": pr.head_branch, + "created_at": pr.created_at.isoformat(), + "updated_at": pr.updated_at.isoformat(), + "state_changed_at": pr.state_changed_at.isoformat() + if pr.state_changed_at + else None, + "commits": pr.commits, + "additions": pr.additions, + "deletions": pr.deletions, + "changed_files": pr.changed_files, + "comments": pr.comments, + "provider": pr.provider, + "first_commit_to_open": pr.first_commit_to_open, + "first_response_time": pr.first_response_time, + "rework_time": pr.rework_time, + "merge_time": pr.merge_time, + "merge_to_deploy": pr.merge_to_deploy, + "cycle_time": pr.cycle_time, + "lead_time": pr.lead_time, + "rework_cycles": pr.rework_cycles, + } + + return pr_data + + +def get_non_paginated_pr_response( + prs: List[PullRequest], + repo_id_map: dict, + total_count: int, + username_user_map: dict = None, +): + username_user_map = username_user_map or {} + return { + "data": [ + { + "id": str(pr.id), + "number": pr.number, + "title": pr.title, + "state": pr.state.value, + "first_commit_to_open": pr.first_commit_to_open, + "merge_to_deploy": pr.merge_to_deploy, + "first_response_time": pr.first_response_time, + "rework_time": pr.rework_time, + "merge_time": pr.merge_time, + "cycle_time": pr.cycle_time, + "lead_time": pr.lead_time, + "author": adapt_user_info(pr.author, username_user_map), + "reviewers": [ + adapt_user_info(r, username_user_map) for r in (pr.reviewers or []) + ], + "repo_name": repo_id_map[pr.repo_id].name, + "pr_link": pr.url, + "base_branch": pr.base_branch, + "head_branch": pr.head_branch, + "created_at": pr.created_at.isoformat(), + "updated_at": pr.updated_at.isoformat(), + "state_changed_at": pr.state_changed_at.isoformat() + if pr.state_changed_at + else None, + "commits": pr.commits, + "additions": pr.additions, + "deletions": pr.deletions, + "changed_files": pr.changed_files, + "comments": pr.comments, + "provider": pr.provider, + "rework_cycles": pr.rework_cycles, + } + for pr in prs + ], + "total_count": total_count, + } + + +def adapt_lead_time_metrics(lead_time_metric: LeadTimeMetrics) -> Dict[str, any]: + return { + "lead_time": lead_time_metric.lead_time, + "first_commit_to_open": lead_time_metric.first_commit_to_open, + "first_response_time": lead_time_metric.first_response_time, + "rework_time": lead_time_metric.rework_time, + "merge_time": lead_time_metric.merge_time, + "merge_to_deploy": lead_time_metric.merge_to_deploy, + "pr_count": lead_time_metric.pr_count, + } diff --git a/backend/analytics_server/dora/api/resources/core_resources.py b/backend/analytics_server/dora/api/resources/core_resources.py new file mode 100644 index 000000000..63330887d --- /dev/null +++ b/backend/analytics_server/dora/api/resources/core_resources.py @@ -0,0 +1,33 @@ +from typing import Dict +from dora.store.models.core.teams import Team + +from dora.store.models import Users + + +def adapt_user_info( + author: str, + username_user_map: Dict[str, Users] = None, +): + if not username_user_map or author not in username_user_map: + return {"username": author, "linked_user": None} + + return { + "username": author, + "linked_user": { + "id": str(username_user_map[author].id), + "name": username_user_map[author].name, + "email": username_user_map[author].primary_email, + "avatar_url": username_user_map[author].avatar_url, + }, + } + + +def adapt_team(team: Team): + return { + "id": str(team.id), + "org_id": str(team.org_id), + "name": team.name, + "member_ids": [str(member_id) for member_id in team.member_ids], + "created_at": team.created_at.isoformat(), + "updated_at": team.updated_at.isoformat(), + } diff --git a/backend/analytics_server/dora/api/resources/deployment_resources.py b/backend/analytics_server/dora/api/resources/deployment_resources.py new file mode 100644 index 000000000..efe401f7a --- /dev/null +++ b/backend/analytics_server/dora/api/resources/deployment_resources.py @@ -0,0 +1,38 @@ +from typing import Dict +from .core_resources import adapt_user_info +from dora.store.models.core.users import Users + +from dora.service.deployments.models.models import ( + Deployment, + DeploymentFrequencyMetrics, +) + + +def adapt_deployment( + deployment: Deployment, username_user_map: Dict[str, Users] = None +) -> Dict: + return { + "id": str(deployment.id), + "deployment_type": deployment.deployment_type.value, + "repo_id": str(deployment.repo_id), + "entity_id": str(deployment.entity_id), + "provider": deployment.provider, + "event_actor": adapt_user_info(deployment.actor, username_user_map), + "head_branch": deployment.head_branch, + "conducted_at": deployment.conducted_at.isoformat(), + "duration": deployment.duration, + "status": deployment.status.value, + "html_url": deployment.html_url, + "meta": deployment.meta, + } + + +def adapt_deployment_frequency_metrics( + deployment_frequency_metrics: DeploymentFrequencyMetrics, +) -> Dict: + return { + "total_deployments": deployment_frequency_metrics.total_deployments, + "avg_daily_deployment_frequency": deployment_frequency_metrics.daily_deployment_frequency, + "avg_weekly_deployment_frequency": deployment_frequency_metrics.avg_weekly_deployment_frequency, + "avg_monthly_deployment_frequency": deployment_frequency_metrics.avg_monthly_deployment_frequency, + } diff --git a/backend/analytics_server/dora/api/resources/incident_resources.py b/backend/analytics_server/dora/api/resources/incident_resources.py new file mode 100644 index 000000000..587ed6042 --- /dev/null +++ b/backend/analytics_server/dora/api/resources/incident_resources.py @@ -0,0 +1,71 @@ +from typing import Dict, List +from dora.service.incidents.models.mean_time_to_recovery import ( + MeanTimeToRecoveryMetrics, + ChangeFailureRateMetrics, +) +from dora.api.resources.deployment_resources import adapt_deployment +from dora.service.deployments.models.models import Deployment +from dora.store.models.incidents import Incident +from dora.api.resources.core_resources import adapt_user_info + + +def adapt_incident( + incident: Incident, + username_user_map: dict = None, +): + return { + "id": str(incident.id), + "title": incident.title, + "key": incident.key, + "incident_number": incident.incident_number, + "provider": incident.provider, + "status": incident.status, + "creation_date": incident.creation_date.isoformat(), + "resolved_date": incident.resolved_date.isoformat() + if incident.resolved_date + else None, + "acknowledged_date": incident.acknowledged_date.isoformat() + if incident.acknowledged_date + else None, + "assigned_to": adapt_user_info(incident.assigned_to, username_user_map), + "assignees": list( + map( + lambda assignee: adapt_user_info(assignee, username_user_map), + incident.assignees or [], + ) + ), + "url": None, # ToDo: Add URL to incidents + "summary": incident.meta.get("summary"), + "incident_type": incident.incident_type.value, + } + + +def adapt_deployments_with_related_incidents( + deployment: Deployment, + deployment_incidents_map: Dict[Deployment, List[Incident]], + username_user_map: dict = None, +): + deployment_response = adapt_deployment(deployment, username_user_map) + incidents = deployment_incidents_map.get(deployment, []) + incident_response = list( + map(lambda incident: adapt_incident(incident, username_user_map), incidents) + ) + deployment_response["incidents"] = incident_response + return deployment_response + + +def adapt_mean_time_to_recovery_metrics( + mean_time_to_recovery: MeanTimeToRecoveryMetrics, +): + return { + "mean_time_to_recovery": mean_time_to_recovery.mean_time_to_recovery, + "incident_count": mean_time_to_recovery.incident_count, + } + + +def adapt_change_failure_rate(change_failure_rate: ChangeFailureRateMetrics): + return { + "change_failure_rate": change_failure_rate.change_failure_rate, + "failed_deployments": change_failure_rate.failed_deployments_count, + "total_deployments": change_failure_rate.total_deployments_count, + } diff --git a/backend/analytics_server/dora/api/resources/settings_resource.py b/backend/analytics_server/dora/api/resources/settings_resource.py new file mode 100644 index 000000000..a932935bd --- /dev/null +++ b/backend/analytics_server/dora/api/resources/settings_resource.py @@ -0,0 +1,61 @@ +from dora.service.settings.models import ( + ConfigurationSettings, + IncidentSettings, + ExcludedPRsSetting, + IncidentTypesSetting, + IncidentSourcesSetting, +) +from dora.store.models import EntityType + + +def adapt_configuration_settings_response(config_settings: ConfigurationSettings): + def _add_entity(config_settings: ConfigurationSettings, response): + + if config_settings.entity_type == EntityType.USER: + response["user_id"] = str(config_settings.entity_id) + + if config_settings.entity_type == EntityType.TEAM: + response["team_id"] = str(config_settings.entity_id) + + if config_settings.entity_type == EntityType.ORG: + response["org_id"] = str(config_settings.entity_id) + + return response + + def _add_setting_data(config_settings: ConfigurationSettings, response): + + # Add new if statements to add settings response for new settings + if isinstance(config_settings.specific_settings, IncidentSettings): + response["setting"] = { + "title_includes": config_settings.specific_settings.title_filters + } + if isinstance(config_settings.specific_settings, ExcludedPRsSetting): + response["setting"] = { + "excluded_pr_ids": config_settings.specific_settings.excluded_pr_ids + } + + if isinstance(config_settings.specific_settings, IncidentTypesSetting): + response["setting"] = { + "incident_types": [ + incident_type.value + for incident_type in config_settings.specific_settings.incident_types + ] + } + + if isinstance(config_settings.specific_settings, IncidentSourcesSetting): + response["setting"] = { + "incident_sources": [ + source.value + for source in config_settings.specific_settings.incident_sources + ] + } + + return response + + response = { + "created_at": config_settings.created_at.isoformat(), + "updated_at": config_settings.updated_at.isoformat(), + } + response = _add_entity(config_settings, response) + response = _add_setting_data(config_settings, response) + return response diff --git a/backend/analytics_server/dora/api/settings.py b/backend/analytics_server/dora/api/settings.py new file mode 100644 index 000000000..179ef94eb --- /dev/null +++ b/backend/analytics_server/dora/api/settings.py @@ -0,0 +1,172 @@ +from typing import Dict + +from flask import Blueprint +from voluptuous import Required, Schema, Coerce, All, Optional +from werkzeug.exceptions import BadRequest, NotFound + +from dora.api.request_utils import dataschema, queryschema, uuid_validator +from dora.api.resources.settings_resource import adapt_configuration_settings_response +from dora.service.query_validator import get_query_validator +from dora.service.settings import get_settings_service, settings_type_validator +from dora.store.models import Organization, Users, SettingType, EntityType + +app = Blueprint("settings", __name__) + + +@app.route("/teams//settings", methods={"GET"}) +@queryschema( + Schema( + { + Required("setting_type"): All(str, Coerce(settings_type_validator)), + Optional("setter_id"): All(str, Coerce(uuid_validator)), + } + ), +) +def get_team_settings(team_id: str, setting_type: SettingType, setter_id: str = None): + + query_validator = get_query_validator() + + team = query_validator.team_validator(team_id) + + setter = None + + if setter_id: + setter = query_validator.user_validator(setter_id) + + if setter and str(setter.org_id) != str(team.org_id): + raise BadRequest(f"User {setter_id} does not belong to team {team_id}") + + settings_service = get_settings_service() + settings = settings_service.get_settings( + setting_type=setting_type, + entity_type=EntityType.TEAM, + entity_id=team_id, + ) + + if not settings: + settings = settings_service.save_settings( + setting_type=setting_type, + entity_type=EntityType.TEAM, + entity_id=team_id, + setter=setter, + ) + + return adapt_configuration_settings_response(settings) + + +@app.route("/teams//settings", methods={"PUT"}) +@dataschema( + Schema( + { + Required("setting_type"): All(str, Coerce(settings_type_validator)), + Optional("setter_id"): All(str, Coerce(uuid_validator)), + Required("setting_data"): dict, + } + ), +) +def put_team_settings( + team_id: str, + setting_type: SettingType, + setter_id: str = None, + setting_data: Dict = None, +): + + query_validator = get_query_validator() + + team = query_validator.team_validator(team_id) + + setter = None + + if setter_id: + setter = query_validator.user_validator(setter_id) + + if setter and str(setter.org_id) != str(team.org_id): + raise BadRequest(f"User {setter_id} does not belong to team {team_id}") + + settings_service = get_settings_service() + settings = settings_service.save_settings( + setting_type=setting_type, + entity_type=EntityType.TEAM, + entity_id=team_id, + setter=setter, + setting_data=setting_data, + ) + return adapt_configuration_settings_response(settings) + + +@app.route("/orgs//settings", methods={"GET"}) +@queryschema( + Schema( + { + Required("setting_type"): All(str, Coerce(settings_type_validator)), + Optional("setter_id"): All(str, Coerce(uuid_validator)), + } + ), +) +def get_org_settings(org_id: str, setting_type: SettingType, setter_id: str = None): + + query_validator = get_query_validator() + org: Organization = query_validator.org_validator(org_id) + + setter = None + + if setter_id: + setter = query_validator.user_validator(setter_id) + + if setter and str(setter.org_id) != str(org_id): + raise BadRequest(f"User {setter_id} does not belong to org {org_id}") + + settings_service = get_settings_service() + settings = settings_service.get_settings( + setting_type=setting_type, + entity_type=EntityType.ORG, + entity_id=org_id, + ) + + if not settings: + settings = settings_service.save_settings( + setting_type=setting_type, + entity_type=EntityType.ORG, + entity_id=org_id, + setter=setter, + ) + + return adapt_configuration_settings_response(settings) + + +@app.route("/orgs//settings", methods={"PUT"}) +@dataschema( + Schema( + { + Required("setting_type"): All(str, Coerce(settings_type_validator)), + Optional("setter_id"): All(str, Coerce(uuid_validator)), + Required("setting_data"): dict, + } + ), +) +def put_org_settings( + org_id: str, + setting_type: SettingType, + setter_id: str = None, + setting_data: Dict = None, +): + query_validator = get_query_validator() + org: Organization = query_validator.org_validator(org_id) + + setter = None + + if setter_id: + setter = query_validator.user_validator(setter_id) + + if setter and str(setter.org_id) != str(org_id): + raise BadRequest(f"User {setter_id} does not belong to org {org_id}") + + settings_service = get_settings_service() + settings = settings_service.save_settings( + setting_type=setting_type, + entity_type=EntityType.ORG, + entity_id=org_id, + setter=setter, + setting_data=setting_data, + ) + return adapt_configuration_settings_response(settings) diff --git a/backend/analytics_server/dora/api/sync.py b/backend/analytics_server/dora/api/sync.py new file mode 100644 index 000000000..ce75d2bd6 --- /dev/null +++ b/backend/analytics_server/dora/api/sync.py @@ -0,0 +1,12 @@ +from flask import Blueprint + +from dora.service.sync_data import trigger_data_sync +from dora.utils.time import time_now + +app = Blueprint("sync", __name__) + + +@app.route("/sync", methods=["POST"]) +def sync(): + trigger_data_sync() + return {"message": "sync started", "time": time_now().isoformat()} diff --git a/backend/analytics_server/dora/api/teams.py b/backend/analytics_server/dora/api/teams.py new file mode 100644 index 000000000..b2b484dc5 --- /dev/null +++ b/backend/analytics_server/dora/api/teams.py @@ -0,0 +1,79 @@ +from flask import Blueprint +from typing import List +from voluptuous import Required, Schema, Optional +from dora.api.resources.core_resources import adapt_team +from dora.store.models.core.teams import Team +from dora.service.core.teams import get_team_service + +from dora.api.request_utils import dataschema +from dora.service.query_validator import get_query_validator + +app = Blueprint("teams", __name__) + + +@app.route("/team/", methods={"GET"}) +def fetch_team(team_id): + + query_validator = get_query_validator() + team: Team = query_validator.team_validator(team_id) + + return adapt_team(team) + + +@app.route("/team/", methods={"PATCH"}) +@dataschema( + Schema( + { + Optional("name"): str, + Optional("member_ids"): list, + } + ), +) +def update_team_patch(team_id: str, name: str = None, member_ids: List[str] = None): + + query_validator = get_query_validator() + team: Team = query_validator.team_validator(team_id) + + if member_ids: + query_validator.users_validator(member_ids) + + team_service = get_team_service() + + team: Team = team_service.update_team(team_id, name, member_ids) + + return adapt_team(team) + + +@app.route("/org//team", methods={"POST"}) +@dataschema( + Schema( + { + Required("name"): str, + Required("member_ids"): list, + } + ), +) +def create_team(org_id: str, name: str, member_ids: List[str]): + + query_validator = get_query_validator() + query_validator.org_validator(org_id) + query_validator.users_validator(member_ids) + + team_service = get_team_service() + + team: Team = team_service.create_team(org_id, name, member_ids) + + return adapt_team(team) + + +@app.route("/team/", methods={"DELETE"}) +def delete_team(team_id: str): + + query_validator = get_query_validator() + team: Team = query_validator.team_validator(team_id) + + team_service = get_team_service() + + team = team_service.delete_team(team_id) + + return adapt_team(team) diff --git a/backend/analytics_server/dora/config/config.ini b/backend/analytics_server/dora/config/config.ini new file mode 100644 index 000000000..75da19b09 --- /dev/null +++ b/backend/analytics_server/dora/config/config.ini @@ -0,0 +1,3 @@ +[KEYS] +SECRET_PRIVATE_KEY = SECRET_PRIVATE_KEY +SECRET_PUBLIC_KEY = SECRET_PUBLIC_KEY \ No newline at end of file diff --git a/backend/analytics_server/dora/exapi/__init__.py b/backend/analytics_server/dora/exapi/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/dora/exapi/git_incidents.py b/backend/analytics_server/dora/exapi/git_incidents.py new file mode 100644 index 000000000..8eb0fb501 --- /dev/null +++ b/backend/analytics_server/dora/exapi/git_incidents.py @@ -0,0 +1,74 @@ +from datetime import datetime +from typing import List + +from dora.exapi.models.git_incidents import RevertPRMap +from dora.service.settings import SettingsService, get_settings_service +from dora.store.models import SettingType, EntityType +from dora.store.models.code import PullRequest, PullRequestRevertPRMapping +from dora.store.models.incidents import IncidentSource +from dora.store.repos.code import CodeRepoService + + +class GitIncidentsAPIService: + def __init__( + self, code_repo_service: CodeRepoService, settings_service: SettingsService + ): + self.code_repo_service = code_repo_service + self.settings_service = settings_service + + def is_sync_enabled(self, org_id: str): + setting = self.settings_service.get_settings( + setting_type=SettingType.INCIDENT_SOURCES_SETTING, + entity_type=EntityType.ORG, + entity_id=org_id, + ) + if setting: + incident_sources_setting = setting.specific_settings + else: + incident_sources_setting = self.settings_service.get_default_setting( + SettingType.INCIDENT_SOURCES_SETTING + ) + incident_sources = incident_sources_setting.incident_sources + return IncidentSource.GIT_REPO in incident_sources + + def get_org_repos(self, org_id: str): + return self.code_repo_service.get_active_org_repos(org_id) + + def get_org_repo(self, repo_id: str): + return self.code_repo_service.get_repo_by_id(repo_id) + + def get_repo_revert_prs_in_interval( + self, repo_id: str, from_time: datetime, to_time: datetime + ) -> List[RevertPRMap]: + revert_pr_mappings: List[ + PullRequestRevertPRMapping + ] = self.code_repo_service.get_repo_revert_prs_mappings_updated_in_interval( + repo_id, from_time, to_time + ) + + revert_pr_ids = [str(pr.pr_id) for pr in revert_pr_mappings] + original_pr_ids = [str(pr.reverted_pr) for pr in revert_pr_mappings] + prs: List[PullRequest] = self.code_repo_service.get_prs_by_ids( + revert_pr_ids + original_pr_ids + ) + id_to_pr_map = {str(pr.id): pr for pr in prs} + + revert_prs: List[RevertPRMap] = [] + for mapping in revert_pr_mappings: + revert_pr = id_to_pr_map.get(str(mapping.pr_id)) + original_pr = id_to_pr_map.get(str(mapping.reverted_pr)) + if revert_pr and original_pr: + revert_prs.append( + RevertPRMap( + revert_pr=revert_pr, + original_pr=original_pr, + created_at=mapping.created_at, + updated_at=mapping.updated_at, + ) + ) + + return revert_prs + + +def get_git_incidents_api_service(): + return GitIncidentsAPIService(CodeRepoService(), get_settings_service()) diff --git a/backend/analytics_server/dora/exapi/github.py b/backend/analytics_server/dora/exapi/github.py new file mode 100644 index 000000000..6ee5c0d25 --- /dev/null +++ b/backend/analytics_server/dora/exapi/github.py @@ -0,0 +1,254 @@ +import contextlib +from datetime import datetime +from http import HTTPStatus +from typing import Optional, Dict, Tuple, List + +import requests +from github import Github, UnknownObjectException +from github.GithubException import RateLimitExceededException +from github.Organization import Organization as GithubOrganization +from github.PaginatedList import PaginatedList as GithubPaginatedList +from github.PullRequest import PullRequest as GithubPullRequest +from github.Repository import Repository as GithubRepository + +from dora.exapi.models.github import GitHubContributor +from dora.utils.log import LOG + +PAGE_SIZE = 100 + + +class GithubRateLimitExceeded(Exception): + pass + + +class GithubApiService: + def __init__(self, access_token: str): + self._token = access_token + self._g = Github(self._token, per_page=PAGE_SIZE) + self.base_url = "https://api.github.com" + self.headers = {"Authorization": f"Bearer {self._token}"} + + @contextlib.contextmanager + def temp_config(self, per_page: int = 30): + self._g.per_page = per_page + yield + self._g.per_page = PAGE_SIZE + + def check_pat(self) -> bool: + """ + Checks if PAT is Valid + :returns: + :raises HTTPError: If the request fails and status code is not 200 + """ + url = f"{self.base_url}/user" + response = requests.get(url, headers=self.headers) + return response.status_code == 200 + + def get_org_list(self) -> [GithubOrganization]: + try: + orgs = list(self._g.get_user().get_orgs()) + except RateLimitExceededException: + raise GithubRateLimitExceeded("GITHUB_API_LIMIT_EXCEEDED") + + return orgs + + def get_repos( + self, org_login: str, per_page: int = 30, page: int = 0 + ) -> [GithubRepository]: + with self.temp_config( + per_page=per_page + ): # This works on assumption of single thread, else make thread local + o = self._g.get_organization(org_login) + repos = o.get_repos().get_page(page) + return repos + + def get_repos_raw( + self, org_login: str, per_page: int = 30, page: int = 0 + ) -> [Dict]: + try: + repos = self.get_repos(org_login, per_page, page) + except RateLimitExceededException: + raise GithubRateLimitExceeded("GITHUB_API_LIMIT_EXCEEDED") + + return [repo.__dict__["_rawData"] for repo in repos] + + def get_repo(self, org_login: str, repo_name: str) -> Optional[GithubRepository]: + try: + return self._g.get_repo(f"{org_login}/{repo_name}") + except UnknownObjectException: + return None + + def get_repo_contributors(self, github_repo: GithubRepository) -> [Tuple[str, int]]: + contributors = list(github_repo.get_contributors()) + return [(u.login, u.contributions) for u in contributors] + + def get_pull_requests( + self, repo: GithubRepository, state="all", sort="updated", direction="desc" + ) -> GithubPaginatedList: + return repo.get_pulls(state=state, sort=sort, direction=direction) + + def get_raw_prs(self, prs: [GithubPullRequest]): + return [pr.__dict__["_rawData"] for pr in prs] + + def get_pull_request( + self, github_repo: GithubRepository, number: int + ) -> GithubPullRequest: + return github_repo.get_pull(number=number) + + def get_pr_commits(self, pr: GithubPullRequest): + return pr.get_commits() + + def get_pr_reviews(self, pr: GithubPullRequest) -> GithubPaginatedList: + return pr.get_reviews() + + def get_contributors( + self, org_login: str, repo_name: str + ) -> List[GitHubContributor]: + + gh_contributors_list = [] + page = 1 + + def _get_contributor_data_from_dict(contributor) -> GitHubContributor: + return GitHubContributor( + login=contributor["login"], + id=contributor["id"], + node_id=contributor["node_id"], + avatar_url=contributor["avatar_url"], + contributions=contributor["contributions"], + events_url=contributor["events_url"], + followers_url=contributor["followers_url"], + following_url=contributor["following_url"], + site_admin=contributor["site_admin"], + gists_url=contributor["gists_url"], + gravatar_id=contributor["gravatar_id"], + html_url=contributor["html_url"], + organizations_url=contributor["organizations_url"], + received_events_url=contributor["received_events_url"], + repos_url=contributor["repos_url"], + starred_url=contributor["starred_url"], + type=contributor["type"], + subscriptions_url=contributor["subscriptions_url"], + url=contributor["url"], + ) + + def _fetch_contributors(page: int = 0): + github_url = f"{self.base_url}/repos/{org_login}/{repo_name}/contributors" + query_params = dict(per_page=PAGE_SIZE, page=page) + response = requests.get( + github_url, headers=self.headers, params=query_params + ) + assert response.status_code == HTTPStatus.OK + return response.json() + + data = _fetch_contributors(page=page) + while data: + gh_contributors_list += data + if len(data) < PAGE_SIZE: + break + + page += 1 + data = _fetch_contributors(page=page) + + contributors: List[GitHubContributor] = [ + _get_contributor_data_from_dict(contributor) + for contributor in gh_contributors_list + ] + return contributors + + def get_org_members(self, org_login: str) -> List[GitHubContributor]: + + gh_org_member_list = [] + page = 1 + + def _get_contributor_data_from_dict(contributor) -> GitHubContributor: + return GitHubContributor( + login=contributor["login"], + id=contributor["id"], + node_id=contributor["node_id"], + avatar_url=contributor["avatar_url"], + events_url=contributor["events_url"], + followers_url=contributor["followers_url"], + following_url=contributor["following_url"], + site_admin=contributor["site_admin"], + gists_url=contributor["gists_url"], + gravatar_id=contributor["gravatar_id"], + html_url=contributor["html_url"], + organizations_url=contributor["organizations_url"], + received_events_url=contributor["received_events_url"], + repos_url=contributor["repos_url"], + starred_url=contributor["starred_url"], + type=contributor["type"], + subscriptions_url=contributor["subscriptions_url"], + url=contributor["url"], + contributions=0, + ) + + def _fetch_members(page: int = 0): + github_url = f"{self.base_url}/orgs/{org_login}/members" + query_params = dict(per_page=PAGE_SIZE, page=page) + response = requests.get( + github_url, headers=self.headers, params=query_params + ) + assert response.status_code == HTTPStatus.OK + return response.json() + + data = _fetch_members(page=page) + while data: + gh_org_member_list += data + if len(data) < PAGE_SIZE: + break + + page += 1 + data = _fetch_members(page=page) + + members: List[GitHubContributor] = [ + _get_contributor_data_from_dict(contributor) + for contributor in gh_org_member_list + ] + return members + + def get_repo_workflows( + self, org_login: str, repo_name: str + ) -> Optional[GithubPaginatedList]: + try: + return self._g.get_repo(f"{org_login}/{repo_name}").get_workflows() + except UnknownObjectException: + return None + + def get_workflow_runs( + self, org_login: str, repo_name: str, workflow_id: str, bookmark: datetime + ): + repo_workflows = [] + page = 1 + + def _fetch_workflow_runs(page: int = 1): + github_url = f"{self.base_url}/repos/{org_login}/{repo_name}/actions/workflows/{workflow_id}/runs" + query_params = dict( + per_page=PAGE_SIZE, + page=page, + created=f"created:>={bookmark.isoformat()}", + ) + response = requests.get( + github_url, headers=self.headers, params=query_params + ) + + if response.status_code == HTTPStatus.NOT_FOUND: + LOG.error( + f"[GitHub Sync Repo Workflow Worker] Workflow {workflow_id} Not found " + f"for repo {org_login}/{repo_name}" + ) + return {} + + assert response.status_code == HTTPStatus.OK + return response.json() + + data = _fetch_workflow_runs(page=page) + while data and data.get("workflow_runs"): + curr_workflow_repos = data.get("workflow_runs") + repo_workflows += curr_workflow_repos + if len(curr_workflow_repos) == 0: + break + + page += 1 + data = _fetch_workflow_runs(page=page) + return repo_workflows diff --git a/backend/analytics_server/dora/exapi/models/__init__.py b/backend/analytics_server/dora/exapi/models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/dora/exapi/models/git_incidents.py b/backend/analytics_server/dora/exapi/models/git_incidents.py new file mode 100644 index 000000000..4931741d2 --- /dev/null +++ b/backend/analytics_server/dora/exapi/models/git_incidents.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass +from datetime import datetime + +from dora.store.models.code import PullRequest + + +@dataclass +class RevertPRMap: + revert_pr: PullRequest + original_pr: PullRequest + created_at: datetime + updated_at: datetime diff --git a/backend/analytics_server/dora/exapi/models/github.py b/backend/analytics_server/dora/exapi/models/github.py new file mode 100644 index 000000000..ee0057f45 --- /dev/null +++ b/backend/analytics_server/dora/exapi/models/github.py @@ -0,0 +1,45 @@ +from dataclasses import dataclass + + +@dataclass +class GitHubBaseUser: + login: str = "" + id: int = 0 + node_id: str = "" + avatar_url: str = "" + gravatar_id: str = "" + url: str = "" + html_url: str = "" + followers_url: str = "" + following_url: str = "" + gists_url: str = "" + starred_url: str = "" + subscriptions_url: str = "" + organizations_url: str = "" + repos_url: str = "" + events_url: str = "" + received_events_url: str = "" + type: str = "User" + site_admin: bool = False + contributions: int = 0 + + def __hash__(self): + return hash(self.id) + + def __eq__(self, other): + if isinstance(other, GitHubBaseUser): + return self.id == other.id + return False + + +@dataclass +class GitHubContributor(GitHubBaseUser): + contributions: int = 0 + + def __hash__(self): + return hash(self.id) + + def __eq__(self, other): + if isinstance(other, GitHubContributor): + return self.id == other.id + return False diff --git a/backend/analytics_server/dora/service/__init__.py b/backend/analytics_server/dora/service/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/dora/service/code/__init__.py b/backend/analytics_server/dora/service/code/__init__.py new file mode 100644 index 000000000..5166d910c --- /dev/null +++ b/backend/analytics_server/dora/service/code/__init__.py @@ -0,0 +1,3 @@ +from .sync import sync_code_repos +from .integration import get_code_integration_service +from .pr_filter import apply_pr_filter diff --git a/backend/analytics_server/dora/service/code/integration.py b/backend/analytics_server/dora/service/code/integration.py new file mode 100644 index 000000000..7e195a05d --- /dev/null +++ b/backend/analytics_server/dora/service/code/integration.py @@ -0,0 +1,27 @@ +from typing import List + +from dora.store.models import UserIdentityProvider, Integration +from dora.store.repos.core import CoreRepoService + +CODE_INTEGRATION_BUCKET = [ + UserIdentityProvider.GITHUB.value, +] + + +class CodeIntegrationService: + def __init__(self, core_repo_service: CoreRepoService): + self.core_repo_service = core_repo_service + + def get_org_providers(self, org_id: str) -> List[str]: + integrations: List[ + Integration + ] = self.core_repo_service.get_org_integrations_for_names( + org_id, CODE_INTEGRATION_BUCKET + ) + if not integrations: + return [] + return [integration.name for integration in integrations] + + +def get_code_integration_service(): + return CodeIntegrationService(core_repo_service=CoreRepoService()) diff --git a/backend/analytics_server/dora/service/code/lead_time.py b/backend/analytics_server/dora/service/code/lead_time.py new file mode 100644 index 000000000..30fe32afc --- /dev/null +++ b/backend/analytics_server/dora/service/code/lead_time.py @@ -0,0 +1,264 @@ +from typing import Dict, List +from datetime import datetime +from dora.service.code.models.lead_time import LeadTimeMetrics +from dora.store.models.code.repository import TeamRepos + +from dora.store.models.code import PRFilter, PullRequest +from dora.store.repos.code import CodeRepoService +from dora.store.models.core import Team + +from dora.service.deployments.deployment_service import ( + DeploymentsService, + get_deployments_service, +) + +from dora.utils.time import ( + Interval, + fill_missing_week_buckets, + generate_expanded_buckets, +) + + +class LeadTimeService: + def __init__( + self, + code_repo_service: CodeRepoService, + deployments_service: DeploymentsService, + ): + self._code_repo_service = code_repo_service + self._deployments_service = deployments_service + + def get_team_lead_time_metrics( + self, + team: Team, + interval: Interval, + pr_filter: Dict[str, PRFilter] = None, + ) -> LeadTimeMetrics: + + team_repos = self._code_repo_service.get_active_team_repos_by_team_id(team.id) + + return self._get_weighted_avg_lead_time_metrics( + self._get_team_repos_lead_time_metrics(team_repos, interval, pr_filter) + ) + + def get_team_lead_time_metrics_trends( + self, + team: Team, + interval: Interval, + pr_filter: Dict[str, PRFilter] = None, + ) -> Dict[datetime, LeadTimeMetrics]: + + team_repos = self._code_repo_service.get_active_team_repos_by_team_id(team.id) + + lead_time_metrics: List[LeadTimeMetrics] = list( + set(self._get_team_repos_lead_time_metrics(team_repos, interval, pr_filter)) + ) + + weekly_lead_time_metrics_map: Dict[ + datetime, List[LeadTimeMetrics] + ] = generate_expanded_buckets( + lead_time_metrics, interval, "merged_at", "weekly" + ) + + weekly_lead_time_metrics_avg_map: Dict[ + datetime, LeadTimeMetrics + ] = self.get_avg_lead_time_metrics_from_map(weekly_lead_time_metrics_map) + + weekly_lead_time_metrics_avg_map = fill_missing_week_buckets( + weekly_lead_time_metrics_avg_map, interval, LeadTimeMetrics + ) + + return weekly_lead_time_metrics_avg_map + + def get_avg_lead_time_metrics_from_map( + self, map_lead_time_metrics: Dict[datetime, List[LeadTimeMetrics]] + ) -> Dict[datetime, LeadTimeMetrics]: + map_avg_lead_time_metrics = {} + for key, lead_time_metrics in map_lead_time_metrics.items(): + map_avg_lead_time_metrics[key] = self._get_weighted_avg_lead_time_metrics( + lead_time_metrics + ) + return map_avg_lead_time_metrics + + def get_team_lead_time_prs( + self, + team: Team, + interval: Interval, + pr_filter: PRFilter = None, + ) -> List[PullRequest]: + + team_repos = self._code_repo_service.get_active_team_repos_by_team_id(team.id) + + ( + team_repos_using_workflow_deployments, + team_repos_using_pr_deployments, + ) = self._deployments_service.get_filtered_team_repos_by_deployment_config( + team_repos + ) + + lead_time_prs_using_workflow = ( + self._get_lead_time_prs_for_repos_using_workflow_deployments( + team_repos_using_workflow_deployments, interval, pr_filter + ) + ) + + lead_time_prs_using_pr = self._get_lead_time_prs_for_repos_using_pr_deployments( + team_repos_using_pr_deployments, interval, pr_filter + ) + + return list(set(lead_time_prs_using_workflow + lead_time_prs_using_pr)) + + def _get_team_repos_lead_time_metrics( + self, + team_repos: TeamRepos, + interval: Interval, + pr_filter: Dict[str, PRFilter] = None, + ) -> List[LeadTimeMetrics]: + + ( + team_repos_using_workflow_deployments, + team_repos_using_pr_deployments, + ) = self._deployments_service.get_filtered_team_repos_by_deployment_config( + team_repos + ) + + lead_time_metrics_using_workflow = ( + self._get_lead_time_metrics_for_repos_using_workflow_deployments( + team_repos_using_workflow_deployments, interval, pr_filter + ) + ) + + lead_time_metrics_using_pr = ( + self._get_lead_time_metrics_for_repos_using_pr_deployments( + team_repos_using_pr_deployments, interval, pr_filter + ) + ) + + return lead_time_metrics_using_workflow + lead_time_metrics_using_pr + + def _get_lead_time_metrics_for_repos_using_workflow_deployments( + self, + team_repos: List[TeamRepos], + interval: Interval, + pr_filter: PRFilter = None, + ) -> List[LeadTimeMetrics]: + + prs = self._get_lead_time_prs_for_repos_using_workflow_deployments( + team_repos, interval, pr_filter + ) + + pr_lead_time_metrics = [self._get_lead_time_metrics_for_pr(pr) for pr in prs] + + return pr_lead_time_metrics + + def _get_lead_time_metrics_for_repos_using_pr_deployments( + self, + team_repos: List[TeamRepos], + interval: Interval, + pr_filter: PRFilter = None, + ) -> Dict[TeamRepos, List[LeadTimeMetrics]]: + + prs = self._get_lead_time_prs_for_repos_using_pr_deployments( + team_repos, interval, pr_filter + ) + + pr_lead_time_metrics = [self._get_lead_time_metrics_for_pr(pr) for pr in prs] + + for prm in pr_lead_time_metrics: + prm.merge_to_deploy = 0 + + return pr_lead_time_metrics + + def _get_lead_time_prs_for_repos_using_workflow_deployments( + self, + team_repos: List[TeamRepos], + interval: Interval, + pr_filter: PRFilter = None, + ) -> List[PullRequest]: + + team_repos_with_workflow_deployments_configured: List[ + TeamRepos + ] = self._deployments_service.get_filtered_team_repos_with_workflow_configured_deployments( + team_repos + ) + + repo_ids = [ + tr.org_repo_id for tr in team_repos_with_workflow_deployments_configured + ] + + prs = self._code_repo_service.get_prs_merged_in_interval( + repo_ids, + interval, + pr_filter, + has_non_null_mtd=True, + ) + + return prs + + def _get_lead_time_prs_for_repos_using_pr_deployments( + self, + team_repos: List[TeamRepos], + interval: Interval, + pr_filter: PRFilter = None, + ) -> List[PullRequest]: + repo_ids = [tr.org_repo_id for tr in team_repos] + + prs = self._code_repo_service.get_prs_merged_in_interval( + repo_ids, interval, pr_filter + ) + + return prs + + def _get_lead_time_metrics_for_pr(self, pr: PullRequest) -> LeadTimeMetrics: + return LeadTimeMetrics( + first_commit_to_open=pr.first_commit_to_open + if pr.first_commit_to_open is not None and pr.first_commit_to_open > 0 + else 0, + first_response_time=pr.first_response_time if pr.first_response_time else 0, + rework_time=pr.rework_time if pr.rework_time else 0, + merge_time=pr.merge_time if pr.merge_time else 0, + merge_to_deploy=pr.merge_to_deploy if pr.merge_to_deploy else 0, + pr_count=1, + merged_at=pr.state_changed_at, + pr_id=pr.id, + ) + + def _get_weighted_avg_lead_time_metrics( + self, lead_time_metrics: List[LeadTimeMetrics] + ) -> LeadTimeMetrics: + return LeadTimeMetrics( + first_commit_to_open=self._get_avg_time( + lead_time_metrics, "first_commit_to_open" + ), + first_response_time=self._get_avg_time( + lead_time_metrics, "first_response_time" + ), + rework_time=self._get_avg_time(lead_time_metrics, "rework_time"), + merge_time=self._get_avg_time(lead_time_metrics, "merge_time"), + merge_to_deploy=self._get_avg_time(lead_time_metrics, "merge_to_deploy"), + pr_count=sum( + [lead_time_metric.pr_count for lead_time_metric in lead_time_metrics] + ), + ) + + def _get_avg_time( + self, lead_time_metrics: List[LeadTimeMetrics], field: str + ) -> float: + total_pr_count = sum( + [lead_time_metric.pr_count for lead_time_metric in lead_time_metrics] + ) + if total_pr_count == 0: + return 0 + + weighted_sum = sum( + [ + getattr(lead_time_metric, field) * lead_time_metric.pr_count + for lead_time_metric in lead_time_metrics + ] + ) + avg = weighted_sum / total_pr_count + return avg + + +def get_lead_time_service() -> LeadTimeService: + return LeadTimeService(CodeRepoService(), get_deployments_service()) diff --git a/backend/analytics_server/dora/service/code/models/lead_time.py b/backend/analytics_server/dora/service/code/models/lead_time.py new file mode 100644 index 000000000..cd0f8ea4d --- /dev/null +++ b/backend/analytics_server/dora/service/code/models/lead_time.py @@ -0,0 +1,49 @@ +from dataclasses import dataclass +from typing import Optional +from datetime import datetime + + +@dataclass +class LeadTimeMetrics: + first_commit_to_open: float = 0 + first_response_time: float = 0 + rework_time: float = 0 + merge_time: float = 0 + merge_to_deploy: float = 0 + pr_count: float = 0 + + merged_at: Optional[datetime] = None + pr_id: Optional[str] = None + + def __eq__(self, other): + if not isinstance(other, LeadTimeMetrics): + raise ValueError( + f"Cannot compare type: LeadTimeMetrics with type: {type(other)}" + ) + if self.pr_id is None: + raise ValueError("PR ID is None") + return self.pr_id == other.pr_id + + def __hash__(self): + if self.pr_id is None: + raise ValueError("PR ID is None") + return hash(self.pr_id) + + @property + def lead_time(self): + return ( + self.first_commit_to_open + + self.first_response_time + + self.rework_time + + self.merge_time + + self.merge_to_deploy + ) + + @property + def cycle_time(self): + return ( + self.first_response_time + + self.rework_time + + self.merge_time + + self.merge_to_deploy + ) diff --git a/backend/analytics_server/dora/service/code/pr_filter.py b/backend/analytics_server/dora/service/code/pr_filter.py new file mode 100644 index 000000000..120d16028 --- /dev/null +++ b/backend/analytics_server/dora/service/code/pr_filter.py @@ -0,0 +1,113 @@ +from typing import List, Dict, Any + +from dora.service.settings.configuration_settings import get_settings_service +from dora.service.settings.models import ExcludedPRsSetting +from dora.store.models.code import PRFilter +from dora.store.models.settings.configuration_settings import SettingType +from dora.store.models.settings.enums import EntityType +from dora.utils.regex import regex_list + + +def apply_pr_filter( + pr_filter: Dict = None, + entity_type: EntityType = None, + entity_id: str = None, + setting_types: List[SettingType] = None, +) -> PRFilter: + processed_pr_filter: PRFilter = ParsePRFilterProcessor(pr_filter).apply() + setting_service = get_settings_service() + setting_type_to_settings_map: Dict[SettingType, Any] = {} + + if entity_type and entity_id and setting_types: + setting_type_to_settings_map = setting_service.get_settings_map( + entity_id, setting_types, entity_type + ) + + if entity_type and entity_id and setting_types: + processed_pr_filter = ConfigurationsPRFilterProcessor( + entity_type, + entity_id, + processed_pr_filter, + setting_types, + setting_type_to_settings_map, + ).apply() + return processed_pr_filter + + +class ParsePRFilterProcessor: + def __init__(self, pr_filter: Dict = None): + self.pr_filter = pr_filter or {} + + def apply(self) -> PRFilter: + authors: List[str] = self.__parse_pr_authors() + base_branches: List[str] = self.__parse_pr_base_branches() + repo_filters: Dict[str, Dict] = self.__parse_repo_filters() + + return PRFilter( + authors=authors, + base_branches=base_branches, + repo_filters=repo_filters, + ) + + def __parse_pr_authors(self) -> List[str]: + return self.pr_filter.get("authors") + + def __parse_pr_base_branches(self) -> List[str]: + base_branches: List[str] = self.pr_filter.get("base_branches") + if base_branches: + base_branches: List[str] = regex_list(base_branches) + return base_branches + + def __parse_repo_filters(self) -> Dict[str, Dict]: + repo_filters: Dict[str, Dict] = self.pr_filter.get("repo_filters") + if repo_filters: + for repo_id, repo_filter in repo_filters.items(): + repo_base_branches: List[str] = self.__parse_repo_base_branches( + repo_filter + ) + repo_filters[repo_id]["base_branches"] = repo_base_branches + return repo_filters + + def __parse_repo_base_branches(self, repo_filter: Dict[str, any]) -> List[str]: + repo_base_branches: List[str] = repo_filter.get("base_branches") + if not repo_base_branches: + return [] + repo_base_branches: List[str] = regex_list(repo_base_branches) + return repo_base_branches + + +class ConfigurationsPRFilterProcessor: + def __init__( + self, + entity_type: EntityType, + entity_id: str, + pr_filter: PRFilter, + setting_types: List[SettingType], + setting_type_to_settings_map: Dict[SettingType, Any] = None, + team_member_usernames: List[str] = None, + ): + self.pr_filter = pr_filter or PRFilter() + self.entity_type: EntityType = entity_type + self.entity_id = entity_id + self.setting_types: List[SettingType] = setting_types or [] + self.setting_type_to_settings_map: Dict[SettingType, Any] = ( + setting_type_to_settings_map or {} + ) + self._setting_service = get_settings_service() + self.team_member_usernames = team_member_usernames or [] + + def apply(self) -> PRFilter: + for setting_type in self.setting_types: + setting = self.setting_type_to_settings_map.get( + setting_type, self._setting_service.get_default_setting(setting_type) + ) + if setting_type == SettingType.EXCLUDED_PRS_SETTING: + self._apply_excluded_pr_ids_setting(setting=setting) + + return self.pr_filter + + def _apply_excluded_pr_ids_setting(self, setting: ExcludedPRsSetting): + + self.pr_filter.excluded_pr_ids = ( + self.pr_filter.excluded_pr_ids or [] + ) + setting.excluded_pr_ids diff --git a/backend/analytics_server/dora/service/code/sync/__init__.py b/backend/analytics_server/dora/service/code/sync/__init__.py new file mode 100644 index 000000000..621b8a041 --- /dev/null +++ b/backend/analytics_server/dora/service/code/sync/__init__.py @@ -0,0 +1 @@ +from .etl_handler import sync_code_repos diff --git a/backend/analytics_server/dora/service/code/sync/etl_code_analytics.py b/backend/analytics_server/dora/service/code/sync/etl_code_analytics.py new file mode 100644 index 000000000..942d8b574 --- /dev/null +++ b/backend/analytics_server/dora/service/code/sync/etl_code_analytics.py @@ -0,0 +1,173 @@ +from datetime import timedelta +from typing import List + +from dora.service.code.sync.models import PRPerformance +from dora.store.models.code import ( + PullRequest, + PullRequestEvent, + PullRequestCommit, + PullRequestEventState, + PullRequestState, +) +from dora.utils.time import Interval + + +class CodeETLAnalyticsService: + def create_pr_metrics( + self, + pr: PullRequest, + pr_events: List[PullRequestEvent], + pr_commits: List[PullRequestCommit], + ) -> PullRequest: + if pr.state == PullRequestState.OPEN: + return pr + + pr_performance = self.get_pr_performance(pr, pr_events) + + pr.first_response_time = ( + pr_performance.first_review_time + if pr_performance.first_review_time != -1 + else None + ) + pr.rework_time = ( + pr_performance.rework_time if pr_performance.rework_time != -1 else None + ) + pr.merge_time = ( + pr_performance.merge_time if pr_performance.merge_time != -1 else None + ) + pr.cycle_time = ( + pr_performance.cycle_time if pr_performance.cycle_time != -1 else None + ) + pr.reviewers = list( + {e.actor_username for e in pr_events if e.actor_username != pr.author} + ) + + if pr_commits: + pr.rework_cycles = self.get_rework_cycles(pr, pr_events, pr_commits) + pr_commits.sort(key=lambda x: x.created_at) + first_commit_to_open = pr.created_at - pr_commits[0].created_at + if isinstance(first_commit_to_open, timedelta): + pr.first_commit_to_open = first_commit_to_open.total_seconds() + + return pr + + @staticmethod + def get_pr_performance(pr: PullRequest, pr_events: [PullRequestEvent]): + pr_events.sort(key=lambda x: x.created_at) + first_review = pr_events[0] if pr_events else None + approved_reviews = list( + filter( + lambda x: x.data["state"] == PullRequestEventState.APPROVED.value, + pr_events, + ) + ) + blocking_reviews = list( + filter( + lambda x: x.data["state"] != PullRequestEventState.APPROVED.value, + pr_events, + ) + ) + + if not approved_reviews: + rework_time = -1 + else: + if first_review.data["state"] == PullRequestEventState.APPROVED.value: + rework_time = 0 + else: + rework_time = ( + approved_reviews[0].created_at - first_review.created_at + ).total_seconds() + + if pr.state != PullRequestState.MERGED or not approved_reviews: + merge_time = -1 + else: + merge_time = ( + pr.state_changed_at - approved_reviews[0].created_at + ).total_seconds() + # Prevent garbage state when PR is approved post merging + merge_time = -1 if merge_time < 0 else merge_time + + cycle_time = pr.state_changed_at - pr.created_at + if isinstance(cycle_time, timedelta): + cycle_time = cycle_time.total_seconds() + + return PRPerformance( + first_review_time=(first_review.created_at - pr.created_at).total_seconds() + if first_review + else -1, + rework_time=rework_time, + merge_time=merge_time, + cycle_time=cycle_time if pr.state == PullRequestState.MERGED else -1, + blocking_reviews=len(blocking_reviews), + approving_reviews=len(pr_events) - len(blocking_reviews), + requested_reviews=len(pr.requested_reviews), + ) + + @staticmethod + def get_rework_cycles( + pr: PullRequest, + pr_events: [PullRequestEvent], + pr_commits: [PullRequestCommit], + ) -> int: + + if not pr_events: + return 0 + + if not pr_commits: + return 0 + + pr_events.sort(key=lambda x: x.created_at) + pr_commits.sort(key=lambda x: x.created_at) + + first_blocking_review = None + last_relevant_approval_review = None + pr_reviewers = dict.fromkeys(pr.reviewers, True) + + for pr_event in pr_events: + if ( + pr_event.state != PullRequestEventState.APPROVED.value + and pr_reviewers.get(pr_event.actor_username) + and not first_blocking_review + ): + first_blocking_review = pr_event + + if pr_event.state == PullRequestEventState.APPROVED.value: + last_relevant_approval_review = pr_event + break + + if not first_blocking_review: + return 0 + + if not last_relevant_approval_review: + return 0 + + interval = Interval( + first_blocking_review.created_at - timedelta(seconds=1), + last_relevant_approval_review.created_at, + ) + + pr_commits = list( + filter( + lambda x: x.created_at in interval, + pr_commits, + ) + ) + pr_reviewers = dict.fromkeys(pr.reviewers, True) + blocking_reviews = list( + filter( + lambda x: x.state != PullRequestEventState.APPROVED.value + and x.actor_username != pr.author + and pr_reviewers.get(x.actor_username) + and x.created_at in interval, + pr_events, + ) + ) + all_events = sorted(pr_commits + blocking_reviews, key=lambda x: x.created_at) + rework_cycles = 0 + for curr, next_event in zip(all_events[:-1], all_events[1:]): + if isinstance(curr, type(next_event)): + continue + if isinstance(next_event, PullRequestCommit): + rework_cycles += 1 + + return rework_cycles diff --git a/backend/analytics_server/dora/service/code/sync/etl_code_factory.py b/backend/analytics_server/dora/service/code/sync/etl_code_factory.py new file mode 100644 index 000000000..570613af0 --- /dev/null +++ b/backend/analytics_server/dora/service/code/sync/etl_code_factory.py @@ -0,0 +1,13 @@ +from dora.service.code.sync.etl_github_handler import get_github_etl_handler +from dora.service.code.sync.etl_provider_handler import CodeProviderETLHandler +from dora.store.models.code import CodeProvider + + +class CodeETLFactory: + def __init__(self, org_id: str): + self.org_id = org_id + + def __call__(self, provider: str) -> CodeProviderETLHandler: + if provider == CodeProvider.GITHUB.value: + return get_github_etl_handler(self.org_id) + raise NotImplementedError(f"Unknown provider - {provider}") diff --git a/backend/analytics_server/dora/service/code/sync/etl_github_handler.py b/backend/analytics_server/dora/service/code/sync/etl_github_handler.py new file mode 100644 index 000000000..1411dabc5 --- /dev/null +++ b/backend/analytics_server/dora/service/code/sync/etl_github_handler.py @@ -0,0 +1,373 @@ +import uuid +from datetime import datetime +from typing import List, Dict, Optional, Tuple, Set + +import pytz +from github.PaginatedList import PaginatedList as GithubPaginatedList +from github.PullRequest import PullRequest as GithubPullRequest +from github.PullRequestReview import PullRequestReview as GithubPullRequestReview +from github.Repository import Repository as GithubRepository + +from dora.exapi.github import GithubApiService +from dora.service.code.sync.etl_code_analytics import CodeETLAnalyticsService +from dora.service.code.sync.etl_provider_handler import CodeProviderETLHandler +from dora.service.code.sync.revert_prs_github_sync import ( + RevertPRsGitHubSyncHandler, + get_revert_prs_github_sync_handler, +) +from dora.store.models import UserIdentityProvider +from dora.store.models.code import ( + OrgRepo, + Bookmark, + PullRequestState, + PullRequest, + PullRequestCommit, + PullRequestEvent, + PullRequestEventType, + PullRequestRevertPRMapping, + CodeProvider, +) +from dora.store.repos.code import CodeRepoService +from dora.store.repos.core import CoreRepoService +from dora.utils.time import time_now, ISO_8601_DATE_FORMAT + +PR_PROCESSING_CHUNK_SIZE = 100 + + +class GithubETLHandler(CodeProviderETLHandler): + def __init__( + self, + org_id: str, + github_api_service: GithubApiService, + code_repo_service: CodeRepoService, + code_etl_analytics_service: CodeETLAnalyticsService, + github_revert_pr_sync_handler: RevertPRsGitHubSyncHandler, + ): + self.org_id: str = org_id + self._api: GithubApiService = github_api_service + self.code_repo_service: CodeRepoService = code_repo_service + self.code_etl_analytics_service: CodeETLAnalyticsService = ( + code_etl_analytics_service + ) + self.github_revert_pr_sync_handler: RevertPRsGitHubSyncHandler = ( + github_revert_pr_sync_handler + ) + self.provider: str = CodeProvider.GITHUB.value + + def check_pat_validity(self) -> bool: + """ + This method checks if the PAT is valid. + :returns: PAT details + :raises: Exception if PAT is invalid + """ + is_valid = self._api.check_pat() + if not is_valid: + raise Exception("Github Personal Access Token is invalid") + return is_valid + + def get_org_repos(self, org_repos: List[OrgRepo]) -> List[OrgRepo]: + """ + This method returns GitHub repos for Org. + :param org_repos: List of OrgRepo objects + :returns: List of GitHub repos as OrgRepo objects + """ + github_repos: List[GithubRepository] = [ + self._api.get_repo(org_repo.org_name, org_repo.name) + for org_repo in org_repos + ] + return [ + self._process_github_repo(org_repos, github_repo) + for github_repo in github_repos + ] + + def get_repo_pull_requests_data( + self, org_repo: OrgRepo, bookmark: Bookmark + ) -> Tuple[List[PullRequest], List[PullRequestCommit], List[PullRequestEvent]]: + """ + This method returns all pull requests, their Commits and Events of a repo. + :param org_repo: OrgRepo object to get pull requests for + :param bookmark: Bookmark date to get all pull requests after this date + :return: Pull requests, their commits and events + """ + github_repo: GithubRepository = self._api.get_repo( + org_repo.org_name, org_repo.name + ) + github_pull_requests: GithubPaginatedList = self._api.get_pull_requests( + github_repo + ) + + prs_to_process = [] + bookmark_time = datetime.fromisoformat(bookmark.bookmark) + for page in range( + 0, github_pull_requests.totalCount // PR_PROCESSING_CHUNK_SIZE + 1, 1 + ): + prs = github_pull_requests.get_page(page) + if not prs: + break + + if prs[-1].updated_at.astimezone(tz=pytz.UTC) <= bookmark_time: + prs_to_process += [ + pr + for pr in prs + if pr.updated_at.astimezone(tz=pytz.UTC) > bookmark_time + ] + break + + prs_to_process += prs + + filtered_prs: List = [] + for pr in prs_to_process: + state_changed_at = pr.merged_at if pr.merged_at else pr.closed_at + if ( + pr.state.upper() != PullRequestState.OPEN.value + and state_changed_at.astimezone(tz=pytz.UTC) < bookmark_time + ): + continue + if pr not in filtered_prs: + filtered_prs.append(pr) + + filtered_prs = filtered_prs[::-1] + + if not filtered_prs: + print("Nothing to process 🎉") + return [], [], [] + + pull_requests: List[PullRequest] = [] + pr_commits: List[PullRequestCommit] = [] + pr_events: List[PullRequestEvent] = [] + prs_added: Set[int] = set() + + for github_pr in filtered_prs: + if github_pr.number in prs_added: + continue + + pr_model, event_models, pr_commit_models = self.process_pr( + str(org_repo.id), github_pr + ) + pull_requests.append(pr_model) + pr_events += event_models + pr_commits += pr_commit_models + prs_added.add(github_pr.number) + + return pull_requests, pr_commits, pr_events + + def process_pr( + self, repo_id: str, pr: GithubPullRequest + ) -> Tuple[PullRequest, List[PullRequestEvent], List[PullRequestCommit]]: + pr_model: Optional[PullRequest] = self.code_repo_service.get_repo_pr_by_number( + repo_id, pr.number + ) + pr_event_model_list: List[ + PullRequestEvent + ] = self.code_repo_service.get_pr_events(pr_model) + pr_commits_model_list: List = [] + + reviews: List[GithubPullRequestReview] = list(self._api.get_pr_reviews(pr)) + pr_model: PullRequest = self._to_pr_model(pr, pr_model, repo_id, len(reviews)) + pr_events_model_list: List[PullRequestEvent] = self._to_pr_events( + reviews, pr_model, pr_event_model_list + ) + if pr.merged_at: + commits: List[Dict] = list( + map( + lambda x: x.__dict__["_rawData"], list(self._api.get_pr_commits(pr)) + ) + ) + pr_commits_model_list: List[PullRequestCommit] = self._to_pr_commits( + commits, pr_model + ) + + pr_model = self.code_etl_analytics_service.create_pr_metrics( + pr_model, pr_events_model_list, pr_commits_model_list + ) + + return pr_model, pr_events_model_list, pr_commits_model_list + + def get_revert_prs_mapping( + self, prs: List[PullRequest] + ) -> List[PullRequestRevertPRMapping]: + return self.github_revert_pr_sync_handler(prs) + + def _process_github_repo( + self, org_repos: List[OrgRepo], github_repo: GithubRepository + ) -> OrgRepo: + + repo_idempotency_key_id_map = { + org_repo.idempotency_key: str(org_repo.id) for org_repo in org_repos + } + + org_repo = OrgRepo( + id=repo_idempotency_key_id_map.get(str(github_repo.id), uuid.uuid4()), + org_id=self.org_id, + name=github_repo.name, + provider=self.provider, + org_name=github_repo.organization.login, + default_branch=github_repo.default_branch, + language=github_repo.language, + contributors=self._api.get_repo_contributors(github_repo), + idempotency_key=str(github_repo.id), + slug=github_repo.name, + updated_at=time_now(), + ) + return org_repo + + def _to_pr_model( + self, + pr: GithubPullRequest, + pr_model: Optional[PullRequest], + repo_id: str, + review_comments: int = 0, + ) -> PullRequest: + state = self._get_state(pr) + pr_id = pr_model.id if pr_model else uuid.uuid4() + state_changed_at = None + if state != PullRequestState.OPEN: + state_changed_at = ( + pr.merged_at.astimezone(pytz.UTC) + if pr.merged_at + else pr.closed_at.astimezone(pytz.UTC) + ) + + merge_commit_sha: Optional[str] = self._get_merge_commit_sha(pr.raw_data, state) + + return PullRequest( + id=pr_id, + number=str(pr.number), + title=pr.title, + url=pr.html_url, + created_at=pr.created_at.astimezone(pytz.UTC), + updated_at=pr.updated_at.astimezone(pytz.UTC), + state_changed_at=state_changed_at, + state=state, + base_branch=pr.base.ref, + head_branch=pr.head.ref, + author=pr.user.login, + repo_id=repo_id, + data=pr.raw_data, + requested_reviews=[r["login"] for r in pr.raw_data["requested_reviewers"]], + meta=dict( + code_stats=dict( + commits=pr.commits, + additions=pr.additions, + deletions=pr.deletions, + changed_files=pr.changed_files, + comments=review_comments, + ), + user_profile=dict(username=pr.user.login), + ), + provider=UserIdentityProvider.GITHUB.value, + merge_commit_sha=merge_commit_sha, + ) + + @staticmethod + def _get_merge_commit_sha(raw_data: Dict, state: PullRequestState) -> Optional[str]: + if state != PullRequestState.MERGED: + return None + + merge_commit_sha = raw_data.get("merge_commit_sha") + + return merge_commit_sha + + @staticmethod + def _get_state(pr: GithubPullRequest) -> PullRequestState: + if pr.merged_at: + return PullRequestState.MERGED + if pr.closed_at: + return PullRequestState.CLOSED + + return PullRequestState.OPEN + + @staticmethod + def _to_pr_events( + reviews: [GithubPullRequestReview], + pr_model: PullRequest, + pr_events_model: [PullRequestEvent], + ) -> List[PullRequestEvent]: + pr_events: List[PullRequestEvent] = [] + pr_event_id_map = {event.idempotency_key: event.id for event in pr_events_model} + + for review in reviews: + if not review.submitted_at: + continue # Discard incomplete reviews + + actor = review.raw_data.get("user", {}) + username = actor.get("login", "") if actor else "" + + pr_events.append( + PullRequestEvent( + id=pr_event_id_map.get(str(review.id), uuid.uuid4()), + pull_request_id=str(pr_model.id), + type=PullRequestEventType.REVIEW.value, + data=review.raw_data, + created_at=review.submitted_at.astimezone(pytz.UTC), + idempotency_key=str(review.id), + org_repo_id=pr_model.repo_id, + actor_username=username, + ) + ) + return pr_events + + def _to_pr_commits( + self, + commits: List[Dict], + pr_model: PullRequest, + ) -> List[PullRequestCommit]: + """ + Sample commit + + { + 'sha': '123456789098765', + 'commit': { + 'committer': {'name': 'abc', 'email': 'abc@midd.com', 'date': '2022-06-29T10:53:15Z'}, + 'message': '[abc 315] avoid mapping edit state', + } + 'author': {'login': 'abc', 'id': 95607047, 'node_id': 'abc', 'avatar_url': ''}, + 'html_url': 'https://github.com/123456789098765', + } + """ + pr_commits: List[PullRequestCommit] = [] + + for commit in commits: + pr_commits.append( + PullRequestCommit( + hash=commit["sha"], + pull_request_id=str(pr_model.id), + url=commit["html_url"], + data=commit, + message=commit["commit"]["message"], + author=commit["author"]["login"] + if commit.get("author") + else commit["commit"].get("committer", {}).get("email", ""), + created_at=self._dt_from_github_dt_string( + commit["commit"]["committer"]["date"] + ), + org_repo_id=pr_model.repo_id, + ) + ) + return pr_commits + + @staticmethod + def _dt_from_github_dt_string(dt_string: str) -> datetime: + dt_without_timezone = datetime.strptime(dt_string, ISO_8601_DATE_FORMAT) + return dt_without_timezone.replace(tzinfo=pytz.UTC) + + +def get_github_etl_handler(org_id: str) -> GithubETLHandler: + def _get_access_token(): + core_repo_service = CoreRepoService() + access_token = core_repo_service.get_access_token( + org_id, UserIdentityProvider.GITHUB + ) + if not access_token: + raise Exception( + f"Access token not found for org {org_id} and provider {UserIdentityProvider.GITHUB.value}" + ) + return access_token + + return GithubETLHandler( + org_id, + GithubApiService(_get_access_token()), + CodeRepoService(), + CodeETLAnalyticsService(), + get_revert_prs_github_sync_handler(), + ) diff --git a/backend/analytics_server/dora/service/code/sync/etl_handler.py b/backend/analytics_server/dora/service/code/sync/etl_handler.py new file mode 100644 index 000000000..52ae79497 --- /dev/null +++ b/backend/analytics_server/dora/service/code/sync/etl_handler.py @@ -0,0 +1,125 @@ +from datetime import datetime, timedelta +from typing import List + +import pytz + +from dora.service.code.integration import get_code_integration_service +from dora.service.code.sync.etl_code_factory import ( + CodeProviderETLHandler, + CodeETLFactory, +) +from dora.service.merge_to_deploy_broker import ( + get_merge_to_deploy_broker_utils_service, + MergeToDeployBrokerUtils, +) +from dora.store.models.code import OrgRepo, BookmarkType, Bookmark, PullRequest +from dora.store.repos.code import CodeRepoService +from dora.utils.log import LOG + + +class CodeETLHandler: + def __init__( + self, + code_repo_service: CodeRepoService, + etl_service: CodeProviderETLHandler, + mtd_broker: MergeToDeployBrokerUtils, + ): + self.code_repo_service = code_repo_service + self.etl_service = etl_service + self.mtd_broker = mtd_broker + + def sync_org_repos(self, org_id: str): + if not self.etl_service.check_pat_validity(): + LOG.error("Invalid PAT for code provider") + return + org_repos: List[OrgRepo] = self._sync_org_repos(org_id) + for org_repo in org_repos: + try: + self._sync_repo_pull_requests_data(org_repo) + except Exception as e: + LOG.error( + f"Error syncing pull requests for repo {org_repo.name}: {str(e)}" + ) + continue + + def _sync_org_repos(self, org_id: str) -> List[OrgRepo]: + try: + org_repos = self.code_repo_service.get_active_org_repos(org_id) + self.etl_service.get_org_repos(org_repos) + self.code_repo_service.update_org_repos(org_repos) + return org_repos + except Exception as e: + LOG.error(f"Error syncing org repos for org {org_id}: {str(e)}") + raise e + + def _sync_repo_pull_requests_data(self, org_repo: OrgRepo) -> None: + try: + bookmark: Bookmark = self.__get_org_repo_bookmark(org_repo) + ( + pull_requests, + pull_request_commits, + pull_request_events, + ) = self.etl_service.get_repo_pull_requests_data(org_repo, bookmark) + self.code_repo_service.save_pull_requests_data( + pull_requests, pull_request_commits, pull_request_events + ) + if not pull_requests: + return + bookmark.bookmark = ( + pull_requests[-1].state_changed_at.astimezone(tz=pytz.UTC).isoformat() + ) + self.code_repo_service.update_org_repo_bookmark(bookmark) + self.mtd_broker.pushback_merge_to_deploy_bookmark( + str(org_repo.id), pull_requests + ) + self.__sync_revert_prs_mapping(org_repo, pull_requests) + except Exception as e: + LOG.error(f"Error syncing pull requests for repo {org_repo.name}: {str(e)}") + raise e + + def __sync_revert_prs_mapping( + self, org_repo: OrgRepo, prs: List[PullRequest] + ) -> None: + try: + revert_prs_mapping = self.etl_service.get_revert_prs_mapping(prs) + self.code_repo_service.save_revert_pr_mappings(revert_prs_mapping) + except Exception as e: + LOG.error(f"Error syncing revert PRs for repo {org_repo.name}: {str(e)}") + raise e + + def __get_org_repo_bookmark(self, org_repo: OrgRepo, default_sync_days: int = 31): + bookmark = self.code_repo_service.get_org_repo_bookmark( + org_repo, BookmarkType.PR + ) + if not bookmark: + default_pr_bookmark = datetime.now().astimezone(tz=pytz.UTC) - timedelta( + days=default_sync_days + ) + bookmark = Bookmark( + repo_id=org_repo.id, + type=BookmarkType.PR.value, + bookmark=default_pr_bookmark.isoformat(), + ) + return bookmark + + +def sync_code_repos(org_id: str): + code_providers: List[str] = get_code_integration_service().get_org_providers(org_id) + if not code_providers: + LOG.info(f"No code integrations found for org {org_id}") + return + etl_factory = CodeETLFactory(org_id) + + for provider in code_providers: + try: + code_etl_handler = CodeETLHandler( + CodeRepoService(), + etl_factory(provider), + get_merge_to_deploy_broker_utils_service(), + ) + code_etl_handler.sync_org_repos(org_id) + LOG.info(f"Synced org repos for provider {provider}") + except Exception as e: + LOG.error(f"Error syncing org repos for provider {provider}: {str(e)}") + continue + LOG.info(f"Synced all org repos for org {org_id}") diff --git a/backend/analytics_server/dora/service/code/sync/etl_provider_handler.py b/backend/analytics_server/dora/service/code/sync/etl_provider_handler.py new file mode 100644 index 000000000..c378b958c --- /dev/null +++ b/backend/analytics_server/dora/service/code/sync/etl_provider_handler.py @@ -0,0 +1,53 @@ +from abc import ABC, abstractmethod +from typing import List, Tuple + +from dora.store.models.code import ( + OrgRepo, + PullRequest, + PullRequestCommit, + PullRequestEvent, + PullRequestRevertPRMapping, + Bookmark, +) + + +class CodeProviderETLHandler(ABC): + @abstractmethod + def check_pat_validity(self) -> bool: + """ + This method checks if the PAT is valid. + :return: PAT details + :raises: Exception if PAT is invalid + """ + pass + + @abstractmethod + def get_org_repos(self, org_repos: List[OrgRepo]) -> List[OrgRepo]: + """ + This method returns all repos from provider that are in sync and available for the provider in given access token. + :return: List of repos as OrgRepo objects + """ + pass + + @abstractmethod + def get_repo_pull_requests_data( + self, org_repo: OrgRepo, bookmark: Bookmark + ) -> Tuple[List[PullRequest], List[PullRequestCommit], List[PullRequestEvent]]: + """ + This method returns all pull requests, their Commits and Events of a repo. After the bookmark date. + :param org_repo: OrgRepo object to get pull requests for + :param bookmark: Bookmark object to get all pull requests after this date + :return: Pull requests sorted by state_changed_at date, their commits and events + """ + pass + + @abstractmethod + def get_revert_prs_mapping( + self, prs: List[PullRequest] + ) -> List[PullRequestRevertPRMapping]: + """ + This method processes all PRs and returns the mapping of revert PRs with source PRs. + :param prs: List of PRs to process + :return: List of PullRequestRevertPRMapping objects + """ + pass diff --git a/backend/analytics_server/dora/service/code/sync/models.py b/backend/analytics_server/dora/service/code/sync/models.py new file mode 100644 index 000000000..69b2176dc --- /dev/null +++ b/backend/analytics_server/dora/service/code/sync/models.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass + + +@dataclass +class PRPerformance: + first_commit_to_open: int = -1 + first_review_time: int = -1 + rework_time: int = -1 + merge_time: int = -1 + merge_to_deploy: int = -1 + cycle_time: int = -1 + blocking_reviews: int = -1 + approving_reviews: int = -1 + requested_reviews: int = -1 + prs_authored_count: int = -1 + additions: int = -1 + deletions: int = -1 + rework_cycles: int = -1 + lead_time: int = -1 diff --git a/backend/analytics_server/dora/service/code/sync/revert_prs_github_sync.py b/backend/analytics_server/dora/service/code/sync/revert_prs_github_sync.py new file mode 100644 index 000000000..74ddefae9 --- /dev/null +++ b/backend/analytics_server/dora/service/code/sync/revert_prs_github_sync.py @@ -0,0 +1,185 @@ +import re +from datetime import datetime +from typing import List, Set, Dict, Optional + +from dora.store.models.code import ( + PullRequest, + PullRequestRevertPRMapping, + PullRequestRevertPRMappingActorType, +) +from dora.store.repos.code import CodeRepoService +from dora.utils.time import time_now + + +class RevertPRsGitHubSyncHandler: + def __init__( + self, + code_repo_service: CodeRepoService, + ): + self.code_repo_service = code_repo_service + + def __call__(self, *args, **kwargs): + return self.process_revert_prs(*args, **kwargs) + + def process_revert_prs( + self, prs: List[PullRequest] + ) -> List[PullRequestRevertPRMapping]: + revert_prs: List[PullRequest] = [] + original_prs: List[PullRequest] = [] + + for pr in prs: + pr_number = ( + self._get_revert_pr_number(pr.head_branch) if pr.head_branch else None + ) + if pr_number is None: + original_prs.append(pr) + else: + revert_prs.append(pr) + + mappings_of_revert_prs = self._get_revert_pr_mapping_for_revert_prs(revert_prs) + mappings_of_original_prs = self._get_revert_pr_mapping_for_original_prs( + original_prs + ) + revert_pr_mappings = set(mappings_of_original_prs + mappings_of_revert_prs) + + return list(revert_pr_mappings) + + def _get_revert_pr_mapping_for_original_prs( + self, prs: List[PullRequest] + ) -> List[PullRequestRevertPRMapping]: + """ + This function takes a list of PRs and for each PR it tries to + find if that pr has been reverted and by which PR. It is done + by taking repo_id and the pr_number and searching for the + string 'revert-[pr-number]' in the head branch. + """ + + repo_ids: Set[str] = set() + repo_id_to_pr_number_to_id_map: Dict[str, Dict[str, str]] = {} + pr_numbers_match_strings: List[str] = [] + + for pr in prs: + pr_numbers_match_strings.append(f"revert-{pr.number}") + repo_ids.add(str(pr.repo_id)) + + if str(pr.repo_id) not in repo_id_to_pr_number_to_id_map: + repo_id_to_pr_number_to_id_map[str(pr.repo_id)] = {} + + repo_id_to_pr_number_to_id_map[str(pr.repo_id)][str(pr.number)] = pr.id + + if len(pr_numbers_match_strings) == 0: + return [] + + revert_prs: List[ + PullRequest + ] = self.code_repo_service.get_prs_by_head_branch_match_strings( + list(repo_ids), pr_numbers_match_strings + ) + + revert_pr_mappings: List[PullRequestRevertPRMapping] = [] + + for rev_pr in revert_prs: + original_pr_number = self._get_revert_pr_number(rev_pr.head_branch) + if original_pr_number is None: + continue + + repo_key_exists = repo_id_to_pr_number_to_id_map.get(str(rev_pr.repo_id)) + if repo_key_exists is None: + continue + + original_pr_id = repo_id_to_pr_number_to_id_map[str(rev_pr.repo_id)].get( + original_pr_number + ) + if original_pr_id is None: + continue + + revert_pr_mp = PullRequestRevertPRMapping( + pr_id=rev_pr.id, + actor_type=PullRequestRevertPRMappingActorType.SYSTEM, + actor=None, + reverted_pr=original_pr_id, + updated_at=time_now(), + ) + revert_pr_mappings.append(revert_pr_mp) + + return revert_pr_mappings + + def _get_revert_pr_mapping_for_revert_prs( + self, prs: List[PullRequest] + ) -> List[PullRequestRevertPRMapping]: + """ + This function takes a list of pull requests and for each pull request + checks if it is a revert pr or not. If it is a revert pr it tries to + create a mapping of that revert pr with the reverted pr and then returns + a list of those mappings + """ + + revert_pr_numbers: List[str] = [] + repo_ids: Set[str] = set() + repo_id_to_pr_number_to_id_map: Dict[str, Dict[str, str]] = {} + + for pr in prs: + revert_pr_number = self._get_revert_pr_number(pr.head_branch) + if revert_pr_number is None: + continue + + revert_pr_numbers.append(revert_pr_number) + repo_ids.add(str(pr.repo_id)) + + if str(pr.repo_id) not in repo_id_to_pr_number_to_id_map: + repo_id_to_pr_number_to_id_map[str(pr.repo_id)] = {} + + repo_id_to_pr_number_to_id_map[str(pr.repo_id)][ + str(revert_pr_number) + ] = pr.id + + if len(revert_pr_numbers) == 0: + return [] + + reverted_prs: List[ + PullRequest + ] = self.code_repo_service.get_reverted_prs_by_numbers( + list(repo_ids), revert_pr_numbers + ) + + revert_pr_mappings: List[PullRequestRevertPRMapping] = [] + for rev_pr in reverted_prs: + repo_key_exists = repo_id_to_pr_number_to_id_map.get(str(rev_pr.repo_id)) + if repo_key_exists is None: + continue + + original_pr_id = repo_id_to_pr_number_to_id_map[str(rev_pr.repo_id)].get( + str(rev_pr.number) + ) + if original_pr_id is None: + continue + + revert_pr_mp = PullRequestRevertPRMapping( + pr_id=original_pr_id, + actor_type=PullRequestRevertPRMappingActorType.SYSTEM, + actor=None, + reverted_pr=rev_pr.id, + updated_at=datetime.now(), + ) + revert_pr_mappings.append(revert_pr_mp) + + return revert_pr_mappings + + def _get_revert_pr_number(self, head_branch: str) -> Optional[str]: + """ + Function to match the regex pattern "revert-[pr-num]-[branch-name]" and + return the PR number for GitHub. + """ + pattern = r"revert-(\d+)-\w+" + + match = re.search(pattern, head_branch) + + if match: + pr_num = match.group(1) + return pr_num + else: + return None + + +def get_revert_prs_github_sync_handler() -> RevertPRsGitHubSyncHandler: + return RevertPRsGitHubSyncHandler(CodeRepoService()) diff --git a/backend/analytics_server/dora/service/core/teams.py b/backend/analytics_server/dora/service/core/teams.py new file mode 100644 index 000000000..c9b233298 --- /dev/null +++ b/backend/analytics_server/dora/service/core/teams.py @@ -0,0 +1,35 @@ +from typing import List, Optional +from dora.store.models.core.teams import Team +from dora.store.repos.core import CoreRepoService + + +class TeamService: + def __init__(self, core_repo_service: CoreRepoService): + self._core_repo_service = core_repo_service + + def get_team(self, team_id: str) -> Optional[Team]: + return self._core_repo_service.get_team(team_id) + + def delete_team(self, team_id: str) -> Optional[Team]: + return self._core_repo_service.delete_team(team_id) + + def create_team(self, org_id: str, name: str, member_ids: List[str] = None) -> Team: + return self._core_repo_service.create_team(org_id, name, member_ids or []) + + def update_team( + self, team_id: str, name: str = None, member_ids: List[str] = None + ) -> Team: + + team = self._core_repo_service.get_team(team_id) + + if name is not None: + team.name = name + + if member_ids is not None: + team.member_ids = member_ids + + return self._core_repo_service.update_team(team) + + +def get_team_service(): + return TeamService(CoreRepoService()) diff --git a/backend/analytics_server/dora/service/deployments/__init__.py b/backend/analytics_server/dora/service/deployments/__init__.py new file mode 100644 index 000000000..7ef06b7be --- /dev/null +++ b/backend/analytics_server/dora/service/deployments/__init__.py @@ -0,0 +1 @@ +from .deployment_pr_mapper import DeploymentPRMapperService diff --git a/backend/analytics_server/dora/service/deployments/analytics.py b/backend/analytics_server/dora/service/deployments/analytics.py new file mode 100644 index 000000000..f4ef19f00 --- /dev/null +++ b/backend/analytics_server/dora/service/deployments/analytics.py @@ -0,0 +1,257 @@ +from collections import defaultdict +from datetime import datetime +from typing import List, Dict, Tuple + +from dora.utils.dict import ( + get_average_of_dict_values, + get_key_to_count_map_from_key_to_list_map, +) + +from .deployment_service import DeploymentsService, get_deployments_service +from dora.store.models.code.filter import PRFilter +from dora.store.models.code.pull_requests import PullRequest +from dora.store.models.code.repository import TeamRepos +from dora.store.models.code.workflows.filter import WorkflowFilter +from dora.service.deployments.models.models import ( + Deployment, + DeploymentFrequencyMetrics, +) + +from dora.store.repos.code import CodeRepoService +from dora.utils.time import Interval, generate_expanded_buckets + + +class DeploymentAnalyticsService: + def __init__( + self, + deployments_service: DeploymentsService, + code_repo_service: CodeRepoService, + ): + self.deployments_service = deployments_service + self.code_repo_service = code_repo_service + + def get_team_successful_deployments_in_interval_with_related_prs( + self, + team_id: str, + interval: Interval, + pr_filter: PRFilter, + workflow_filter: WorkflowFilter, + ) -> Dict[str, List[Dict[Deployment, List[PullRequest]]]]: + """ + Retrieves successful deployments within the specified interval for a given team, + along with related pull requests. Returns A dictionary mapping repository IDs to lists of deployments along with related pull requests. Each deployment is associated with a list of pull requests that contributed to it. + """ + + deployments: List[ + Deployment + ] = self.deployments_service.get_team_successful_deployments_in_interval( + team_id, interval, pr_filter, workflow_filter + ) + + team_repos: List[TeamRepos] = self._get_team_repos_by_team_id(team_id) + repo_ids: List[str] = [str(team_repo.org_repo_id) for team_repo in team_repos] + + pull_requests: List[ + PullRequest + ] = self.code_repo_service.get_prs_merged_in_interval( + repo_ids, interval, pr_filter + ) + + repo_id_branch_to_pr_list_map: Dict[ + Tuple[str, str], List[PullRequest] + ] = self._map_prs_to_repo_id_and_base_branch(pull_requests) + repo_id_branch_to_deployments_map: Dict[ + Tuple[str, str], List[Deployment] + ] = self._map_deployments_to_repo_id_and_head_branch(deployments) + + repo_id_to_deployments_with_pr_map: Dict[ + str, Dict[Deployment, List[PullRequest]] + ] = defaultdict(dict) + + for ( + repo_id, + base_branch, + ), deployments in repo_id_branch_to_deployments_map.items(): + relevant_prs: List[PullRequest] = repo_id_branch_to_pr_list_map.get( + (repo_id, base_branch), [] + ) + deployments_pr_map: Dict[ + Deployment, List[PullRequest] + ] = self._map_prs_to_deployments(relevant_prs, deployments) + + repo_id_to_deployments_with_pr_map[repo_id].update(deployments_pr_map) + + return repo_id_to_deployments_with_pr_map + + def get_team_deployment_frequency_metrics( + self, + team_id: str, + interval: Interval, + pr_filter: PRFilter, + workflow_filter: WorkflowFilter, + ) -> DeploymentFrequencyMetrics: + + team_successful_deployments = ( + self.deployments_service.get_team_successful_deployments_in_interval( + team_id, interval, pr_filter, workflow_filter + ) + ) + + return self._get_deployment_frequency_metrics( + team_successful_deployments, interval + ) + + def get_weekly_deployment_frequency_trends( + self, + team_id: str, + interval: Interval, + pr_filter: PRFilter, + workflow_filter: WorkflowFilter, + ) -> Dict[datetime, int]: + + team_successful_deployments = ( + self.deployments_service.get_team_successful_deployments_in_interval( + team_id, interval, pr_filter, workflow_filter + ) + ) + + team_weekly_deployments = generate_expanded_buckets( + team_successful_deployments, interval, "conducted_at", "weekly" + ) + + return get_key_to_count_map_from_key_to_list_map(team_weekly_deployments) + + def _map_prs_to_repo_id_and_base_branch( + self, pull_requests: List[PullRequest] + ) -> Dict[Tuple[str, str], List[PullRequest]]: + repo_id_branch_pr_map: Dict[Tuple[str, str], List[PullRequest]] = defaultdict( + list + ) + for pr in pull_requests: + repo_id = str(pr.repo_id) + base_branch = pr.base_branch + repo_id_branch_pr_map[(repo_id, base_branch)].append(pr) + return repo_id_branch_pr_map + + def _map_deployments_to_repo_id_and_head_branch( + self, deployments: List[Deployment] + ) -> Dict[Tuple[str, str], List[Deployment]]: + repo_id_branch_deployments_map: Dict[ + Tuple[str, str], List[Deployment] + ] = defaultdict(list) + for deployment in deployments: + repo_id = str(deployment.repo_id) + head_branch = deployment.head_branch + repo_id_branch_deployments_map[(repo_id, head_branch)].append(deployment) + return repo_id_branch_deployments_map + + def _map_prs_to_deployments( + self, pull_requests: List[PullRequest], deployments: List[Deployment] + ) -> Dict[Deployment, List[PullRequest]]: + """ + Maps the pull requests to the deployments they were included in. + This method takes a sorted list of pull requests and a sorted list of deployments and returns a dictionary + """ + pr_count = 0 + deployment_count = 0 + deployment_pr_map = defaultdict( + list, {deployment: [] for deployment in deployments} + ) + + while pr_count < len(pull_requests) and deployment_count < len(deployments): + pr = pull_requests[pr_count] + deployment = deployments[deployment_count] + + # Check if the PR was merged before or at the same time as the deployment + if pr.state_changed_at <= deployment.conducted_at: + deployment_pr_map[deployment].append(pr) + pr_count += 1 + else: + deployment_count += 1 + + return deployment_pr_map + + def _get_team_repos_by_team_id(self, team_id: str) -> List[TeamRepos]: + return self.code_repo_service.get_active_team_repos_by_team_id(team_id) + + def _get_deployment_frequency_from_date_to_deployment_map( + self, date_to_deployment_map: Dict[datetime, List[Deployment]] + ) -> int: + """ + This method takes a dict of datetime representing (day/week/month) to Deployments and returns avg deployment frequency + """ + + date_to_deployment_count_map: Dict[ + datetime, int + ] = get_key_to_count_map_from_key_to_list_map(date_to_deployment_map) + + return get_average_of_dict_values(date_to_deployment_count_map) + + def _get_deployment_frequency_metrics( + self, successful_deployments: List[Deployment], interval: Interval + ) -> DeploymentFrequencyMetrics: + + successful_deployments = list( + filter( + lambda x: x.conducted_at >= interval.from_time + and x.conducted_at <= interval.to_time, + successful_deployments, + ) + ) + + team_daily_deployments = generate_expanded_buckets( + successful_deployments, interval, "conducted_at", "daily" + ) + team_weekly_deployments = generate_expanded_buckets( + successful_deployments, interval, "conducted_at", "weekly" + ) + team_monthly_deployments = generate_expanded_buckets( + successful_deployments, interval, "conducted_at", "monthly" + ) + + daily_deployment_frequency = ( + self._get_deployment_frequency_from_date_to_deployment_map( + team_daily_deployments + ) + ) + + weekly_deployment_frequency = ( + self._get_deployment_frequency_from_date_to_deployment_map( + team_weekly_deployments + ) + ) + + monthly_deployment_frequency = ( + self._get_deployment_frequency_from_date_to_deployment_map( + team_monthly_deployments + ) + ) + + return DeploymentFrequencyMetrics( + len(successful_deployments), + daily_deployment_frequency, + weekly_deployment_frequency, + monthly_deployment_frequency, + ) + + def _get_weekly_deployment_frequency_trends( + self, successful_deployments: List[Deployment], interval: Interval + ) -> Dict[datetime, int]: + + successful_deployments = list( + filter( + lambda x: x.conducted_at >= interval.from_time + and x.conducted_at <= interval.to_time, + successful_deployments, + ) + ) + + team_weekly_deployments = generate_expanded_buckets( + successful_deployments, interval, "conducted_at", "weekly" + ) + + return get_key_to_count_map_from_key_to_list_map(team_weekly_deployments) + + +def get_deployment_analytics_service() -> DeploymentAnalyticsService: + return DeploymentAnalyticsService(get_deployments_service(), CodeRepoService()) diff --git a/backend/analytics_server/dora/service/deployments/deployment_pr_mapper.py b/backend/analytics_server/dora/service/deployments/deployment_pr_mapper.py new file mode 100644 index 000000000..fa4971e87 --- /dev/null +++ b/backend/analytics_server/dora/service/deployments/deployment_pr_mapper.py @@ -0,0 +1,79 @@ +from collections import defaultdict +from datetime import datetime +from queue import Queue +from typing import List +from dora.store.models.code.enums import PullRequestState + +from dora.store.models.code.pull_requests import PullRequest +from dora.service.deployments.models.models import Deployment + + +class DeploymentPRGraph: + def __init__(self): + self._nodes = set() + self._adj_list = defaultdict(list) + self._last_change_deployed_for_branch = {} + + def add_edge(self, base_branch, head_branch, pr: PullRequest): + self._nodes.add(base_branch) + + self._adj_list[base_branch].append((head_branch, pr)) + if head_branch not in self._last_change_deployed_for_branch: + self._last_change_deployed_for_branch[head_branch] = pr.state_changed_at + + self._last_change_deployed_for_branch[head_branch] = max( + self._last_change_deployed_for_branch[head_branch], pr.state_changed_at + ) + + def get_edges(self, base_branch: str): + return self._adj_list[base_branch] + + def get_all_prs_for_root(self, base_branch) -> List[PullRequest]: + if base_branch not in self._nodes: + return [] + + prs = set() + q = Queue() + visited = defaultdict(bool) + q.put(base_branch) + + while not q.empty(): + front = q.get() + if visited[front]: + continue + + visited[front] = True + for edge in self.get_edges(front): + branch, pr = edge + if self._is_pr_merged_post_last_change(pr=pr, base_branch=front): + continue + + q.put(branch) + prs.add(pr) + + return list(prs) + + def _is_pr_merged_post_last_change(self, pr: PullRequest, base_branch: str): + return pr.state_changed_at > self._last_change_deployed_for_branch[base_branch] + + def set_root_deployment_time(self, root_branch, deployment_time: datetime): + self._last_change_deployed_for_branch[root_branch] = deployment_time + + +class DeploymentPRMapperService: + def get_all_prs_deployed( + self, prs: List[PullRequest], deployment: Deployment + ) -> List[PullRequest]: + + branch_graph = DeploymentPRGraph() + branch_graph.set_root_deployment_time( + deployment.head_branch, deployment.conducted_at + ) + + for pr in prs: + if pr.state != PullRequestState.MERGED: + continue + + branch_graph.add_edge(pr.base_branch, pr.head_branch, pr) + + return branch_graph.get_all_prs_for_root(deployment.head_branch) diff --git a/backend/analytics_server/dora/service/deployments/deployment_service.py b/backend/analytics_server/dora/service/deployments/deployment_service.py new file mode 100644 index 000000000..4278615c1 --- /dev/null +++ b/backend/analytics_server/dora/service/deployments/deployment_service.py @@ -0,0 +1,171 @@ +from typing import List, Tuple +from dora.store.models.code.workflows import RepoWorkflowType, RepoWorkflow + +from .factory import get_deployments_factory +from .deployments_factory_service import DeploymentsFactoryService +from dora.store.models.code.filter import PRFilter +from dora.store.models.code.repository import TeamRepos +from dora.store.models.code.workflows.filter import WorkflowFilter +from dora.service.deployments.models.models import Deployment, DeploymentType + +from dora.store.repos.code import CodeRepoService +from dora.store.repos.workflows import WorkflowRepoService +from dora.utils.time import Interval + + +class DeploymentsService: + def __init__( + self, + code_repo_service: CodeRepoService, + workflow_repo_service: WorkflowRepoService, + workflow_based_deployments_service: DeploymentsFactoryService, + pr_based_deployments_service: DeploymentsFactoryService, + ): + self.code_repo_service = code_repo_service + self.workflow_repo_service = workflow_repo_service + self.workflow_based_deployments_service = workflow_based_deployments_service + self.pr_based_deployments_service = pr_based_deployments_service + + def get_team_successful_deployments_in_interval( + self, + team_id: str, + interval: Interval, + pr_filter: PRFilter = None, + workflow_filter: WorkflowFilter = None, + ) -> List[Deployment]: + team_repos = self._get_team_repos_by_team_id(team_id) + ( + team_repos_using_workflow_deployments, + team_repos_using_pr_deployments, + ) = self.get_filtered_team_repos_by_deployment_config(team_repos) + + deployments_using_workflow = self.workflow_based_deployments_service.get_repos_successful_deployments_in_interval( + self._get_repo_ids_from_team_repos(team_repos_using_workflow_deployments), + interval, + workflow_filter, + ) + deployments_using_pr = self.pr_based_deployments_service.get_repos_successful_deployments_in_interval( + self._get_repo_ids_from_team_repos(team_repos_using_pr_deployments), + interval, + pr_filter, + ) + + deployments: List[Deployment] = ( + deployments_using_workflow + deployments_using_pr + ) + sorted_deployments = self._sort_deployments_by_date(deployments) + + return sorted_deployments + + def get_filtered_team_repos_with_workflow_configured_deployments( + self, team_repos: List[TeamRepos] + ) -> List[TeamRepos]: + """ + Get team repos with workflow deployments configured. + That is the repo has a workflow configured and team repo has deployment type as workflow. + """ + filtered_team_repos: List[ + TeamRepos + ] = self._filter_team_repos_using_workflow_deployments(team_repos) + + repo_ids = [str(tr.org_repo_id) for tr in filtered_team_repos] + repo_id_to_team_repo_map = { + str(tr.org_repo_id): tr for tr in filtered_team_repos + } + + repo_workflows: List[ + RepoWorkflow + ] = self.workflow_repo_service.get_repo_workflow_by_repo_ids( + repo_ids, RepoWorkflowType.DEPLOYMENT + ) + workflows_repo_ids = list( + set([str(workflow.org_repo_id) for workflow in repo_workflows]) + ) + + team_repos_with_workflow_deployments = [ + repo_id_to_team_repo_map[repo_id] + for repo_id in workflows_repo_ids + if repo_id in repo_id_to_team_repo_map + ] + + return team_repos_with_workflow_deployments + + def get_team_all_deployments_in_interval( + self, + team_id: str, + interval, + pr_filter: PRFilter = None, + workflow_filter: WorkflowFilter = None, + ) -> List[Deployment]: + + team_repos = self._get_team_repos_by_team_id(team_id) + ( + team_repos_using_workflow_deployments, + team_repos_using_pr_deployments, + ) = self.get_filtered_team_repos_by_deployment_config(team_repos) + + deployments_using_workflow = self.workflow_based_deployments_service.get_repos_all_deployments_in_interval( + self._get_repo_ids_from_team_repos(team_repos_using_workflow_deployments), + interval, + workflow_filter, + ) + deployments_using_pr = ( + self.pr_based_deployments_service.get_repos_all_deployments_in_interval( + self._get_repo_ids_from_team_repos(team_repos_using_pr_deployments), + interval, + pr_filter, + ) + ) + + deployments: List[Deployment] = ( + deployments_using_workflow + deployments_using_pr + ) + sorted_deployments = self._sort_deployments_by_date(deployments) + + return sorted_deployments + + def _get_team_repos_by_team_id(self, team_id: str) -> List[TeamRepos]: + return self.code_repo_service.get_active_team_repos_by_team_id(team_id) + + def _get_repo_ids_from_team_repos(self, team_repos: List[TeamRepos]) -> List[str]: + return [str(team_repo.org_repo_id) for team_repo in team_repos] + + def get_filtered_team_repos_by_deployment_config( + self, team_repos: List[TeamRepos] + ) -> Tuple[List[TeamRepos], List[TeamRepos]]: + """ + Splits the input TeamRepos list into two TeamRepos List, TeamRepos using workflow and TeamRepos using pr deployments. + """ + return self._filter_team_repos_using_workflow_deployments( + team_repos + ), self._filter_team_repos_using_pr_deployments(team_repos) + + def _filter_team_repos_using_workflow_deployments( + self, team_repos: List[TeamRepos] + ): + return [ + team_repo + for team_repo in team_repos + if team_repo.deployment_type.value == DeploymentType.WORKFLOW.value + ] + + def _filter_team_repos_using_pr_deployments(self, team_repos: List[TeamRepos]): + return [ + team_repo + for team_repo in team_repos + if team_repo.deployment_type.value == DeploymentType.PR_MERGE.value + ] + + def _sort_deployments_by_date( + self, deployments: List[Deployment] + ) -> List[Deployment]: + return sorted(deployments, key=lambda deployment: deployment.conducted_at) + + +def get_deployments_service() -> DeploymentsService: + return DeploymentsService( + CodeRepoService(), + WorkflowRepoService(), + get_deployments_factory(DeploymentType.WORKFLOW), + get_deployments_factory(DeploymentType.PR_MERGE), + ) diff --git a/backend/analytics_server/dora/service/deployments/deployments_factory_service.py b/backend/analytics_server/dora/service/deployments/deployments_factory_service.py new file mode 100644 index 000000000..95d5316e8 --- /dev/null +++ b/backend/analytics_server/dora/service/deployments/deployments_factory_service.py @@ -0,0 +1,46 @@ +from abc import ABC, abstractmethod +from typing import List, Dict, Tuple +from urllib.parse import unquote +from dora.store.models.code.pull_requests import PullRequest +from uuid import UUID +from werkzeug.exceptions import BadRequest + +from dora.store.models.code.repository import TeamRepos +from dora.service.deployments.models.models import Deployment, DeploymentType + + +class DeploymentsFactoryService(ABC): + @abstractmethod + def get_repos_successful_deployments_in_interval( + self, repo_ids, interval, specific_filter + ) -> List[Deployment]: + pass + + @abstractmethod + def get_repos_all_deployments_in_interval( + self, repo_ids, interval, specific_filter + ) -> List[Deployment]: + pass + + @abstractmethod + def get_pull_requests_related_to_deployment( + self, deployment: Deployment + ) -> List[PullRequest]: + pass + + @abstractmethod + def get_deployment_by_entity_id(self, entity_id: str) -> Deployment: + pass + + @classmethod + def get_deployment_type_and_entity_id_from_deployment_id( + cls, id_str: str + ) -> Tuple[DeploymentType, str]: + id_str = unquote(id_str) + # Split the id string by '|' + deployment_type, entity_id = id_str.split("|") + try: + UUID(entity_id) + except ValueError: + raise BadRequest(f"Invalid UUID entity id: {entity_id}") + return DeploymentType(deployment_type), entity_id diff --git a/backend/analytics_server/dora/service/deployments/factory.py b/backend/analytics_server/dora/service/deployments/factory.py new file mode 100644 index 000000000..fba02ec27 --- /dev/null +++ b/backend/analytics_server/dora/service/deployments/factory.py @@ -0,0 +1,27 @@ +from .models.adapter import DeploymentsAdaptorFactory +from dora.service.deployments.models.models import DeploymentType +from dora.store.repos.code import CodeRepoService +from dora.store.repos.workflows import WorkflowRepoService +from .deployment_pr_mapper import DeploymentPRMapperService +from .deployments_factory_service import DeploymentsFactoryService +from .pr_deployments_service import PRDeploymentsService +from .workflow_deployments_service import WorkflowDeploymentsService + + +def get_deployments_factory( + deployment_type: DeploymentType, +) -> DeploymentsFactoryService: + if deployment_type == DeploymentType.PR_MERGE: + return PRDeploymentsService( + CodeRepoService(), + DeploymentsAdaptorFactory(DeploymentType.PR_MERGE).get_adaptor(), + ) + elif deployment_type == DeploymentType.WORKFLOW: + return WorkflowDeploymentsService( + WorkflowRepoService(), + CodeRepoService(), + DeploymentsAdaptorFactory(DeploymentType.WORKFLOW).get_adaptor(), + DeploymentPRMapperService(), + ) + else: + raise ValueError(f"Unknown deployment type: {deployment_type}") diff --git a/backend/analytics_server/dora/service/deployments/models/__init__.py b/backend/analytics_server/dora/service/deployments/models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/dora/service/deployments/models/adapter.py b/backend/analytics_server/dora/service/deployments/models/adapter.py new file mode 100644 index 000000000..17397bf04 --- /dev/null +++ b/backend/analytics_server/dora/service/deployments/models/adapter.py @@ -0,0 +1,105 @@ +from abc import ABC +from typing import Union, List, Tuple +from dora.store.models.code.enums import PullRequestState +from dora.store.models.code.pull_requests import PullRequest + +from dora.store.models.code.workflows.workflows import RepoWorkflow, RepoWorkflowRuns +from dora.service.deployments.models.models import ( + Deployment, + DeploymentStatus, + DeploymentType, +) + + +class DeploymentsAdaptor(ABC): + def adapt(self, entity: Union[Tuple[RepoWorkflow, RepoWorkflowRuns], PullRequest]): + pass + + def adapt_many( + self, entities: List[Union[Tuple[RepoWorkflow, RepoWorkflowRuns], PullRequest]] + ): + pass + + +class DeploymentsAdaptorFactory: + def __init__(self, deployment_type: DeploymentType): + self.deployment_type = deployment_type + + def get_adaptor(self) -> DeploymentsAdaptor: + if self.deployment_type == DeploymentType.WORKFLOW: + return WorkflowRunsToDeploymentsAdaptor() + elif self.deployment_type == DeploymentType.PR_MERGE: + return PullRequestToDeploymentsAdaptor() + else: + raise ValueError( + f"Unsupported deployment type: {self.deployment_type.value}" + ) + + +class WorkflowRunsToDeploymentsAdaptor(DeploymentsAdaptor): + def adapt(self, entity: Tuple[RepoWorkflow, RepoWorkflowRuns]): + repo_workflow, repo_workflow_run = entity + return Deployment( + deployment_type=DeploymentType.WORKFLOW, + repo_id=str(repo_workflow.org_repo_id), + entity_id=str(repo_workflow_run.id), + provider=repo_workflow.provider.value, + actor=repo_workflow_run.event_actor, + head_branch=repo_workflow_run.head_branch, + conducted_at=repo_workflow_run.conducted_at, + duration=repo_workflow_run.duration, + status=DeploymentStatus(repo_workflow_run.status.value), + html_url=repo_workflow_run.html_url, + meta=dict( + id=str(repo_workflow.id), + repo_workflow_id=str(repo_workflow_run.repo_workflow_id), + provider_workflow_run_id=repo_workflow_run.provider_workflow_run_id, + event_actor=repo_workflow_run.event_actor, + head_branch=repo_workflow_run.head_branch, + status=repo_workflow_run.status.value, + conducted_at=repo_workflow_run.conducted_at.isoformat(), + duration=repo_workflow_run.duration, + html_url=repo_workflow_run.html_url, + ), + ) + + def adapt_many(self, entities: List[Tuple[RepoWorkflow, RepoWorkflowRuns]]): + return [self.adapt(entity) for entity in entities] + + +class PullRequestToDeploymentsAdaptor(DeploymentsAdaptor): + def adapt(self, entity: PullRequest): + if not self._is_pull_request_merged(entity): + raise ValueError("Pull request is not merged") + return Deployment( + deployment_type=DeploymentType.PR_MERGE, + repo_id=str(entity.repo_id), + entity_id=str(entity.id), + provider=entity.provider, + actor=entity.username, + head_branch=entity.base_branch, + conducted_at=entity.state_changed_at, + duration=0, + status=DeploymentStatus.SUCCESS, + html_url=entity.url, + meta=dict( + id=str(entity.id), + repo_id=str(entity.repo_id), + number=entity.number, + provider=entity.provider, + username=entity.username, + base_branch=entity.base_branch, + state_changed_at=entity.state_changed_at.isoformat(), + url=entity.url, + ), + ) + + def adapt_many(self, entities: List[PullRequest]): + return [ + self.adapt(entity) + for entity in entities + if self._is_pull_request_merged(entity) + ] + + def _is_pull_request_merged(self, entity: PullRequest): + return entity.state == PullRequestState.MERGED diff --git a/backend/analytics_server/dora/service/deployments/models/models.py b/backend/analytics_server/dora/service/deployments/models/models.py new file mode 100644 index 000000000..d3a4676f1 --- /dev/null +++ b/backend/analytics_server/dora/service/deployments/models/models.py @@ -0,0 +1,46 @@ +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from voluptuous import default_factory + + +class DeploymentType(Enum): + WORKFLOW = "WORKFLOW" + PR_MERGE = "PR_MERGE" + + +class DeploymentStatus(Enum): + SUCCESS = "SUCCESS" + FAILURE = "FAILURE" + PENDING = "PENDING" + CANCELLED = "CANCELLED" + + +@dataclass +class Deployment: + deployment_type: DeploymentType + repo_id: str + entity_id: str + provider: str + actor: str + head_branch: str + conducted_at: datetime + duration: int + status: DeploymentStatus + html_url: str + meta: dict = default_factory(dict) + + def __hash__(self): + return hash(self.deployment_type.value + "|" + str(self.entity_id)) + + @property + def id(self): + return self.deployment_type.value + "|" + str(self.entity_id) + + +@dataclass +class DeploymentFrequencyMetrics: + total_deployments: int + daily_deployment_frequency: int + avg_weekly_deployment_frequency: int + avg_monthly_deployment_frequency: int diff --git a/backend/analytics_server/dora/service/deployments/pr_deployments_service.py b/backend/analytics_server/dora/service/deployments/pr_deployments_service.py new file mode 100644 index 000000000..f3292b8a6 --- /dev/null +++ b/backend/analytics_server/dora/service/deployments/pr_deployments_service.py @@ -0,0 +1,51 @@ +from typing import List +from .models.adapter import DeploymentsAdaptor +from dora.store.models.code.filter import PRFilter +from dora.store.models.code.pull_requests import PullRequest +from dora.service.deployments.models.models import Deployment + +from dora.store.repos.code import CodeRepoService +from dora.utils.time import Interval + +from .deployments_factory_service import DeploymentsFactoryService + + +class PRDeploymentsService(DeploymentsFactoryService): + def __init__( + self, + code_repo_service: CodeRepoService, + deployments_adapter: DeploymentsAdaptor, + ): + self.code_repo_service = code_repo_service + self.deployments_adapter = deployments_adapter + + def get_repos_successful_deployments_in_interval( + self, repo_ids: List[str], interval: Interval, pr_filter: PRFilter + ) -> List[Deployment]: + pull_requests: List[ + PullRequest + ] = self.code_repo_service.get_prs_merged_in_interval( + repo_ids, interval, pr_filter=pr_filter + ) + + return self.deployments_adapter.adapt_many(pull_requests) + + def get_repos_all_deployments_in_interval( + self, repo_ids: List[str], interval: Interval, prs_filter: PRFilter + ) -> List[Deployment]: + return self.get_repos_successful_deployments_in_interval( + repo_ids, interval, prs_filter + ) + + def get_pull_requests_related_to_deployment( + self, deployment: Deployment + ) -> List[PullRequest]: + return [self.code_repo_service.get_pull_request_by_id(deployment.entity_id)] + + def get_deployment_by_entity_id(self, entity_id: str) -> Deployment: + pull_request: PullRequest = self.code_repo_service.get_pull_request_by_id( + entity_id + ) + if not pull_request: + raise ValueError(f"Pull Request with id {entity_id} not found") + return self.deployments_adapter.adapt(pull_request) diff --git a/backend/analytics_server/dora/service/deployments/workflow_deployments_service.py b/backend/analytics_server/dora/service/deployments/workflow_deployments_service.py new file mode 100644 index 000000000..2d91258cc --- /dev/null +++ b/backend/analytics_server/dora/service/deployments/workflow_deployments_service.py @@ -0,0 +1,92 @@ +from typing import List, Tuple +from .models.adapter import DeploymentsAdaptor +from dora.store.models.code.pull_requests import PullRequest +from dora.store.models.code.repository import TeamRepos +from dora.store.models.code.workflows.filter import WorkflowFilter +from dora.store.models.code.workflows.workflows import RepoWorkflow, RepoWorkflowRuns +from dora.service.deployments.models.models import Deployment +from dora.store.repos.code import CodeRepoService + +from dora.store.repos.workflows import WorkflowRepoService +from dora.utils.time import Interval + +from .deployment_pr_mapper import DeploymentPRMapperService +from .deployments_factory_service import DeploymentsFactoryService + + +class WorkflowDeploymentsService(DeploymentsFactoryService): + def __init__( + self, + workflow_repo_service: WorkflowRepoService, + code_repo_service: CodeRepoService, + deployments_adapter: DeploymentsAdaptor, + deployment_pr_mapping_service: DeploymentPRMapperService, + ): + self.workflow_repo_service = workflow_repo_service + self.code_repo_service = code_repo_service + self.deployments_adapter = deployments_adapter + self.deployment_pr_mapping_service = deployment_pr_mapping_service + + def get_repos_successful_deployments_in_interval( + self, repo_ids: List[str], interval: Interval, workflow_filter: WorkflowFilter + ) -> List[Deployment]: + repo_workflow_runs: List[ + Tuple[RepoWorkflow, RepoWorkflowRuns] + ] = self.workflow_repo_service.get_successful_repo_workflows_runs_by_repo_ids( + repo_ids, interval, workflow_filter + ) + return self.deployments_adapter.adapt_many(repo_workflow_runs) + + def get_repos_all_deployments_in_interval( + self, + repo_ids: List[str], + interval: Interval, + workflow_filter: WorkflowFilter, + ) -> List[Deployment]: + repo_workflow_runs: List[ + Tuple[RepoWorkflow, RepoWorkflowRuns] + ] = self.workflow_repo_service.get_repos_workflow_runs_by_repo_ids( + repo_ids, interval, workflow_filter + ) + return self.deployments_adapter.adapt_many(repo_workflow_runs) + + def get_pull_requests_related_to_deployment( + self, deployment: Deployment + ) -> List[PullRequest]: + previous_deployment = self._get_previous_deployment_for_given_deployment( + deployment + ) + interval = Interval(previous_deployment.conducted_at, deployment.conducted_at) + pr_base_branch: str = deployment.head_branch + pull_requests: List[ + PullRequest + ] = self.code_repo_service.get_prs_merged_in_interval( + [deployment.repo_id], interval, base_branches=[pr_base_branch] + ) + relevant_prs: List[ + PullRequest + ] = self.deployment_pr_mapping_service.get_all_prs_deployed( + pull_requests, deployment + ) + + return relevant_prs + + def get_deployment_by_entity_id(self, entity_id: str) -> Deployment: + repo_workflow_run: Tuple[ + RepoWorkflow, RepoWorkflowRuns + ] = self.workflow_repo_service.get_repo_workflow_run_by_id(entity_id) + if not repo_workflow_run: + raise ValueError(f"Workflow run with id {entity_id} not found") + return self.deployments_adapter.adapt(repo_workflow_run) + + def _get_previous_deployment_for_given_deployment( + self, deployment: Deployment + ) -> Deployment: + ( + workflow_run, + current_workflow_run, + ) = self.workflow_repo_service.get_repo_workflow_run_by_id(deployment.entity_id) + workflow_run_previous_workflow_run: Tuple[ + RepoWorkflow, RepoWorkflowRuns + ] = self.workflow_repo_service.get_previous_workflow_run(current_workflow_run) + return self.deployments_adapter.adapt(workflow_run_previous_workflow_run) diff --git a/backend/analytics_server/dora/service/external_integrations_service.py b/backend/analytics_server/dora/service/external_integrations_service.py new file mode 100644 index 000000000..38e8c2fab --- /dev/null +++ b/backend/analytics_server/dora/service/external_integrations_service.py @@ -0,0 +1,63 @@ +from github.Organization import Organization as GithubOrganization + +from dora.exapi.github import GithubApiService, GithubRateLimitExceeded +from dora.store.models import UserIdentityProvider +from dora.store.repos.core import CoreRepoService + +PAGE_SIZE = 100 + + +class ExternalIntegrationsService: + def __init__( + self, + org_id: str, + user_identity_provider: UserIdentityProvider, + access_token: str, + ): + self.org_id = org_id + self.user_identity_provider = user_identity_provider + self.access_token = access_token + + def get_github_organizations(self): + github_api_service = GithubApiService(self.access_token) + try: + orgs: [GithubOrganization] = github_api_service.get_org_list() + except GithubRateLimitExceeded as e: + raise Exception(e) + return orgs + + def get_github_org_repos(self, org_login: str, page_size: int, page: int): + github_api_service = GithubApiService(self.access_token) + try: + return github_api_service.get_repos_raw(org_login, page_size, page) + except Exception as e: + raise Exception(e) + + def get_repo_workflows(self, gh_org_name: str, gh_org_repo_name: str): + github_api_service = GithubApiService(self.access_token) + workflows = github_api_service.get_repo_workflows(gh_org_name, gh_org_repo_name) + workflows_list = [] + for page in range(0, workflows.totalCount // PAGE_SIZE + 1, 1): + workflows = workflows.get_page(page) + if not workflows: + break + workflows_list += workflows + return workflows_list + + +def get_external_integrations_service( + org_id: str, user_identity_provider: UserIdentityProvider +): + def _get_access_token() -> str: + access_token = CoreRepoService().get_access_token( + org_id, user_identity_provider + ) + if not access_token: + raise Exception( + f"Access token not found for org {org_id} and provider {user_identity_provider.value}" + ) + return access_token + + return ExternalIntegrationsService( + org_id, user_identity_provider, _get_access_token() + ) diff --git a/backend/analytics_server/dora/service/incidents/__init__.py b/backend/analytics_server/dora/service/incidents/__init__.py new file mode 100644 index 000000000..242f1f6fb --- /dev/null +++ b/backend/analytics_server/dora/service/incidents/__init__.py @@ -0,0 +1 @@ +from .sync import sync_org_incidents diff --git a/backend/analytics_server/dora/service/incidents/incident_filter.py b/backend/analytics_server/dora/service/incidents/incident_filter.py new file mode 100644 index 000000000..43cfd7b3f --- /dev/null +++ b/backend/analytics_server/dora/service/incidents/incident_filter.py @@ -0,0 +1,109 @@ +from typing import Dict, List, Any, Optional +from dora.store.models.settings.configuration_settings import SettingType +from dora.service.settings.configuration_settings import ( + get_settings_service, + IncidentSettings, + IncidentTypesSetting, +) +from dora.store.models.incidents import IncidentFilter + +from dora.store.models.settings import EntityType + + +class IncidentFilterService: + def __init__( + self, + raw_incident_filter: Dict = None, + entity_type: EntityType = None, + entity_id: str = None, + setting_types: List[SettingType] = None, + setting_type_to_settings_map: Dict[SettingType, Any] = None, + ): + self.raw_incident_filter: Dict = raw_incident_filter or {} + self.entity_type: EntityType = entity_type + self.entity_id = entity_id + self.setting_types: List[SettingType] = setting_types or [] + self.setting_type_to_settings_map: Dict[SettingType, any] = ( + setting_type_to_settings_map or {} + ) + + def apply(self): + incident_filter: IncidentFilter = IncidentFilter() + if self.entity_type and self.entity_id: + incident_filter = ConfigurationsIncidentFilterProcessor( + incident_filter, + self.entity_type, + self.entity_id, + self.setting_types, + self.setting_type_to_settings_map, + ).apply() + return incident_filter + + +def apply_incident_filter( + incident_filter: Dict = None, + entity_type: EntityType = None, + entity_id: str = None, + setting_types: List[SettingType] = None, +) -> IncidentFilter: + setting_service = get_settings_service() + setting_type_to_settings_map = setting_service.get_settings_map( + entity_id, setting_types, entity_type + ) + + return IncidentFilterService( + incident_filter, + entity_type, + entity_id, + setting_types, + setting_type_to_settings_map, + ).apply() + + +class ConfigurationsIncidentFilterProcessor: + def __init__( + self, + incident_filter: IncidentFilter, + entity_type: EntityType, + entity_id: str, + setting_types: List[SettingType], + setting_type_to_settings_map: Dict[SettingType, Any], + ): + self.incident_filter = incident_filter or IncidentFilter() + self.entity_type: EntityType = entity_type + self.entity_id = entity_id + self.setting_types: List[SettingType] = setting_types or [] + self.setting_type_to_settings_map = setting_type_to_settings_map + + def apply(self): + if SettingType.INCIDENT_SETTING in self.setting_types: + self.incident_filter.title_filter_substrings = ( + self.__incident_title_filter() + ) + + if SettingType.INCIDENT_TYPES_SETTING in self.setting_types: + self.incident_filter.incident_types = self.__incident_type_setting() + + return self.incident_filter + + def __incident_title_filter(self) -> List[str]: + setting: Optional[IncidentSettings] = self.setting_type_to_settings_map.get( + SettingType.INCIDENT_SETTING + ) + if not setting: + return [] + title_filters = [] + if setting and isinstance(setting, IncidentSettings): + title_filters = setting.title_filters + return title_filters + + def __incident_type_setting(self) -> List[str]: + setting: Optional[IncidentTypesSetting] = self.setting_type_to_settings_map.get( + SettingType.INCIDENT_TYPES_SETTING + ) + if not setting: + return [] + incident_types = [] + if setting and isinstance(setting, IncidentTypesSetting): + incident_types = setting.incident_types + return incident_types diff --git a/backend/analytics_server/dora/service/incidents/incidents.py b/backend/analytics_server/dora/service/incidents/incidents.py new file mode 100644 index 000000000..6a3ccc1d8 --- /dev/null +++ b/backend/analytics_server/dora/service/incidents/incidents.py @@ -0,0 +1,213 @@ +from collections import defaultdict +from datetime import datetime +from typing import List, Dict, Tuple +from dora.service.incidents.models.mean_time_to_recovery import ( + ChangeFailureRateMetrics, + MeanTimeToRecoveryMetrics, +) +from dora.service.deployments.models.models import Deployment +from dora.service.incidents.incident_filter import apply_incident_filter +from dora.store.models.incidents.filter import IncidentFilter +from dora.store.models.settings import EntityType, SettingType +from dora.utils.time import ( + Interval, + fill_missing_week_buckets, + generate_expanded_buckets, + get_given_weeks_monday, +) + +from dora.store.models.incidents import Incident +from dora.service.settings.configuration_settings import ( + SettingsService, + get_settings_service, +) +from dora.store.repos.incidents import IncidentsRepoService + + +class IncidentService: + def __init__( + self, + incidents_repo_service: IncidentsRepoService, + settings_service: SettingsService, + ): + self._incidents_repo_service = incidents_repo_service + self._settings_service = settings_service + + def get_resolved_team_incidents( + self, team_id: str, interval: Interval + ) -> List[Incident]: + incident_filter: IncidentFilter = apply_incident_filter( + entity_type=EntityType.TEAM, + entity_id=team_id, + setting_types=[ + SettingType.INCIDENT_SETTING, + SettingType.INCIDENT_TYPES_SETTING, + ], + ) + return self._incidents_repo_service.get_resolved_team_incidents( + team_id, interval, incident_filter + ) + + def get_team_incidents(self, team_id: str, interval: Interval) -> List[Incident]: + incident_filter: IncidentFilter = apply_incident_filter( + entity_type=EntityType.TEAM, + entity_id=team_id, + setting_types=[ + SettingType.INCIDENT_SETTING, + SettingType.INCIDENT_TYPES_SETTING, + ], + ) + return self._incidents_repo_service.get_team_incidents( + team_id, interval, incident_filter + ) + + def get_deployment_incidents_map( + self, deployments: List[Deployment], incidents: List[Incident] + ): + deployments = sorted(deployments, key=lambda x: x.conducted_at) + incidents = sorted(incidents, key=lambda x: x.creation_date) + incidents_pointer = 0 + + deployment_incidents_map: Dict[Deployment, List[Incident]] = defaultdict(list) + + for current_deployment, next_deployment in zip( + deployments, deployments[1:] + [None] + ): + current_deployment_incidents = [] + + if incidents_pointer >= len(incidents): + deployment_incidents_map[ + current_deployment + ] = current_deployment_incidents + continue + + while incidents_pointer < len(incidents): + incident = incidents[incidents_pointer] + + if incident.creation_date >= current_deployment.conducted_at and ( + next_deployment is None + or incident.creation_date < next_deployment.conducted_at + ): + current_deployment_incidents.append(incident) + incidents_pointer += 1 + elif incident.creation_date < current_deployment.conducted_at: + incidents_pointer += 1 + else: + break + + deployment_incidents_map[current_deployment] = current_deployment_incidents + + return deployment_incidents_map + + def get_team_mean_time_to_recovery( + self, team_id: str, interval: Interval + ) -> MeanTimeToRecoveryMetrics: + + resolved_team_incidents = self.get_resolved_team_incidents(team_id, interval) + + return self._get_incidents_mean_time_to_recovery(resolved_team_incidents) + + def get_team_mean_time_to_recovery_trends( + self, team_id: str, interval: Interval + ) -> MeanTimeToRecoveryMetrics: + + resolved_team_incidents = self.get_resolved_team_incidents(team_id, interval) + + weekly_resolved_team_incidents: Dict[ + datetime, List[Incident] + ] = generate_expanded_buckets( + resolved_team_incidents, interval, "resolved_date", "weekly" + ) + + weekly_mean_time_to_recovery: Dict[datetime, MeanTimeToRecoveryMetrics] = {} + + for week, incidents in weekly_resolved_team_incidents.items(): + + if incidents: + weekly_mean_time_to_recovery[ + week + ] = self._get_incidents_mean_time_to_recovery(incidents) + else: + weekly_mean_time_to_recovery[week] = MeanTimeToRecoveryMetrics() + + return weekly_mean_time_to_recovery + + def calculate_change_failure_deployments( + self, deployment_incidents_map: Dict[Deployment, List[Incident]] + ) -> Tuple[List[Deployment], List[Deployment]]: + failed_deployments = [ + deployment + for deployment, incidents in deployment_incidents_map.items() + if incidents + ] + all_deployments: List[Deployment] = list(deployment_incidents_map.keys()) + + return failed_deployments, all_deployments + + def get_change_failure_rate_metrics( + self, deployments: List[Deployment], incidents: List[Incident] + ) -> ChangeFailureRateMetrics: + deployment_incidents_map = self.get_deployment_incidents_map( + deployments, incidents + ) + ( + failed_deployments, + all_deployments, + ) = self.calculate_change_failure_deployments(deployment_incidents_map) + return ChangeFailureRateMetrics(set(failed_deployments), set(all_deployments)) + + def get_weekly_change_failure_rate( + self, + interval: Interval, + deployments: List[Deployment], + incidents: List[Incident], + ) -> ChangeFailureRateMetrics: + + deployments_incidents_map = self.get_deployment_incidents_map( + deployments, incidents + ) + week_start_to_change_failure_rate_map: Dict[ + datetime, ChangeFailureRateMetrics + ] = defaultdict(ChangeFailureRateMetrics) + + for deployment, incidents in deployments_incidents_map.items(): + week_start_date = get_given_weeks_monday(deployment.conducted_at) + if incidents: + week_start_to_change_failure_rate_map[ + week_start_date + ].failed_deployments.add(deployment) + week_start_to_change_failure_rate_map[ + week_start_date + ].total_deployments.add(deployment) + + return fill_missing_week_buckets( + week_start_to_change_failure_rate_map, interval, ChangeFailureRateMetrics + ) + + def _calculate_incident_resolution_time(self, incident: Incident) -> int: + return (incident.resolved_date - incident.creation_date).total_seconds() + + def _get_incidents_mean_time_to_recovery( + self, resolved_incidents: List[Incident] + ) -> MeanTimeToRecoveryMetrics: + + incident_count = len(resolved_incidents) + + if not incident_count: + return MeanTimeToRecoveryMetrics() + + mean_time_to_recovery = ( + sum( + [ + self._calculate_incident_resolution_time(incident) + for incident in resolved_incidents + ] + ) + / incident_count + ) + + return MeanTimeToRecoveryMetrics(mean_time_to_recovery, incident_count) + + +def get_incident_service(): + return IncidentService(IncidentsRepoService(), get_settings_service()) diff --git a/backend/analytics_server/dora/service/incidents/integration.py b/backend/analytics_server/dora/service/incidents/integration.py new file mode 100644 index 000000000..5e3632896 --- /dev/null +++ b/backend/analytics_server/dora/service/incidents/integration.py @@ -0,0 +1,65 @@ +from typing import List + +from dora.service.settings import SettingsService, get_settings_service +from dora.service.settings.models import IncidentSourcesSetting +from dora.store.models import Integration, SettingType, EntityType +from dora.store.models.incidents import IncidentProvider, IncidentSource +from dora.store.repos.core import CoreRepoService + +GIT_INCIDENT_INTEGRATION_BUCKET = [IncidentProvider.GITHUB.value] + + +class IncidentsIntegrationService: + def __init__( + self, core_repo_service: CoreRepoService, settings_service: SettingsService + ): + self.core_repo_service = core_repo_service + self.settings_service = settings_service + + def get_org_providers(self, org_id: str) -> List[str]: + integrations: List[ + Integration + ] = self.core_repo_service.get_org_integrations_for_names( + org_id, self._get_possible_incident_providers(org_id) + ) + if not integrations: + return [] + return [integration.name for integration in integrations] + + def _get_possible_incident_providers(self, org_id: str) -> List[str]: + + valid_integration_types = [] + + incident_source_setting: IncidentSourcesSetting = ( + self._get_or_create_incident_source_setting(org_id) + ) + + if IncidentSource.GIT_REPO in incident_source_setting.incident_sources: + valid_integration_types += GIT_INCIDENT_INTEGRATION_BUCKET + + return valid_integration_types + + def _get_or_create_incident_source_setting( + self, org_id: str + ) -> IncidentSourcesSetting: + + settings = self.settings_service.get_settings( + setting_type=SettingType.INCIDENT_SOURCES_SETTING, + entity_type=EntityType.ORG, + entity_id=org_id, + ) + + if not settings: + settings = self.settings_service.save_settings( + setting_type=SettingType.INCIDENT_SOURCES_SETTING, + entity_type=EntityType.ORG, + entity_id=org_id, + ) + return settings.specific_settings + + +def get_incidents_integration_service(): + return IncidentsIntegrationService( + core_repo_service=CoreRepoService(), + settings_service=get_settings_service(), + ) diff --git a/backend/analytics_server/dora/service/incidents/models/mean_time_to_recovery.py b/backend/analytics_server/dora/service/incidents/models/mean_time_to_recovery.py new file mode 100644 index 000000000..509bf2447 --- /dev/null +++ b/backend/analytics_server/dora/service/incidents/models/mean_time_to_recovery.py @@ -0,0 +1,34 @@ +from dataclasses import dataclass +from typing import Optional, Set + +from dora.service.deployments.models.models import Deployment + + +@dataclass +class MeanTimeToRecoveryMetrics: + mean_time_to_recovery: Optional[float] = None + incident_count: int = 0 + + +@dataclass +class ChangeFailureRateMetrics: + failed_deployments: Set[Deployment] = None + total_deployments: Set[Deployment] = None + + def __post_init__(self): + self.failed_deployments = self.failed_deployments or set() + self.total_deployments = self.total_deployments or set() + + @property + def change_failure_rate(self): + if not self.total_deployments: + return 0 + return len(self.failed_deployments) / len(self.total_deployments) * 100 + + @property + def failed_deployments_count(self): + return len(self.failed_deployments) + + @property + def total_deployments_count(self): + return len(self.total_deployments) diff --git a/backend/analytics_server/dora/service/incidents/sync/__init__.py b/backend/analytics_server/dora/service/incidents/sync/__init__.py new file mode 100644 index 000000000..445ee7725 --- /dev/null +++ b/backend/analytics_server/dora/service/incidents/sync/__init__.py @@ -0,0 +1 @@ +from .etl_handler import sync_org_incidents diff --git a/backend/analytics_server/dora/service/incidents/sync/etl_git_incidents_handler.py b/backend/analytics_server/dora/service/incidents/sync/etl_git_incidents_handler.py new file mode 100644 index 000000000..ee01914ea --- /dev/null +++ b/backend/analytics_server/dora/service/incidents/sync/etl_git_incidents_handler.py @@ -0,0 +1,242 @@ +from datetime import datetime +from typing import List, Dict, Optional, Tuple + +from dora.exapi.git_incidents import ( + GitIncidentsAPIService, + get_git_incidents_api_service, +) +from dora.exapi.models.git_incidents import RevertPRMap +from dora.service.incidents.sync.etl_provider_handler import IncidentsProviderETLHandler +from dora.store.models.code import OrgRepo, PullRequest +from dora.store.models.incidents import ( + IncidentSource, + OrgIncidentService, + IncidentType, + IncidentsBookmark, + IncidentOrgIncidentServiceMap, + IncidentStatus, + Incident, + IncidentProvider, +) +from dora.store.repos.incidents import IncidentsRepoService +from dora.utils.log import LOG +from dora.utils.string import uuid4_str +from dora.utils.time import time_now + + +class GitIncidentsETLHandler(IncidentsProviderETLHandler): + def __init__( + self, + org_id: str, + git_incidents_api_service: GitIncidentsAPIService, + incidents_repo_service: IncidentsRepoService, + ): + self.org_id = org_id + self.git_incidents_api_service = git_incidents_api_service + self.incidents_repo_service = incidents_repo_service + + def check_pat_validity(self) -> bool: + """ + Checks if Incident Source, "GIT_REPO" is enabled for the org + :return: True if enabled, False otherwise + """ + return self.git_incidents_api_service.is_sync_enabled(self.org_id) + + def get_updated_incident_services( + self, incident_services: List[OrgIncidentService] + ) -> List[OrgIncidentService]: + """ + Get the updated Incident Services for the org + :param incident_services: List of Incident Services + :return: List of updated Incident Services + """ + git_repo_type_incident_services = [ + incident_service + for incident_service in incident_services + if incident_service.source_type == IncidentSource.GIT_REPO + ] + active_org_repos: List[OrgRepo] = self.git_incidents_api_service.get_org_repos( + self.org_id + ) + + key_to_service_map: Dict[str, OrgIncidentService] = { + incident_service.key: incident_service + for incident_service in git_repo_type_incident_services + } + + updated_services: List[OrgIncidentService] = [] + + for org_repo in active_org_repos: + updated_services.append( + self._adapt_org_incident_service( + org_repo, key_to_service_map.get(str(org_repo.id)) + ) + ) + + return updated_services + + def process_service_incidents( + self, + incident_service: OrgIncidentService, + bookmark: IncidentsBookmark, + ) -> Tuple[List[Incident], List[IncidentOrgIncidentServiceMap], IncidentsBookmark]: + """ + Sync incidents for the service + :param incident_service: OrgIncidentService + :param bookmark: IncidentsBookmark + :return: List of Incidents, List of IncidentOrgIncidentServiceMap, IncidentsBookmark + """ + if not incident_service or not isinstance(incident_service, OrgIncidentService): + raise Exception(f"Service not found") + + from_time: datetime = bookmark.bookmark + to_time: datetime = time_now() + + revert_pr_incidents: List[ + RevertPRMap + ] = self.git_incidents_api_service.get_repo_revert_prs_in_interval( + incident_service.key, from_time, to_time + ) + if not revert_pr_incidents: + LOG.warning( + f"[GIT Incidents Sync] Incidents not received for service {str(incident_service.id)} " + f"in org {self.org_id} since {from_time.isoformat()}" + ) + return [], [], bookmark + + revert_pr_incidents.sort( + key=lambda revert_pr_incident: revert_pr_incident.updated_at + ) + + bookmark.bookmark = max(bookmark.bookmark, revert_pr_incidents[-1].updated_at) + + incidents, incident_org_incident_service_map_models = self._process_incidents( + incident_service, revert_pr_incidents + ) + + return incidents, incident_org_incident_service_map_models, bookmark + + def _process_incidents( + self, + org_incident_service: OrgIncidentService, + revert_pr_incidents: List[RevertPRMap], + ) -> Tuple[List[Incident], List[IncidentOrgIncidentServiceMap]]: + + if not revert_pr_incidents: + LOG.warning( + f"[GitIncidentsService Incident Sync] Incidents not received for " + f"service {str(org_incident_service.id)} in org {self.org_id}" + ) + return [], [] + + incident_models = [] + incident_org_incident_service_map_models = [] + + for revert_pr_incident in revert_pr_incidents: + try: + incident, incident_service_map = self._process_revert_pr_incident( + org_incident_service, revert_pr_incident + ) + incident_models.append(incident) + incident_org_incident_service_map_models.append(incident_service_map) + except Exception as e: + LOG.error( + f"ERROR processing revert pr Incident in service {str(org_incident_service.id)} in " + f"org {str(org_incident_service.org_id)}, Error: {str(e)}" + ) + raise e + + return incident_models, incident_org_incident_service_map_models + + def _process_revert_pr_incident( + self, org_incident_service: OrgIncidentService, revert_pr_map: RevertPRMap + ) -> Tuple[Incident, IncidentOrgIncidentServiceMap]: + incident_unique_id = str(revert_pr_map.original_pr.id) + existing_incident: Optional[ + Incident + ] = self.incidents_repo_service.get_incident_by_key_type_and_provider( + incident_unique_id, + IncidentType.REVERT_PR, + IncidentProvider(org_incident_service.provider), + ) + incident_id = existing_incident.id if existing_incident else uuid4_str() + + incident = Incident( + id=incident_id, + provider=org_incident_service.provider, + key=str(incident_unique_id), + title=revert_pr_map.original_pr.title, + incident_number=int(revert_pr_map.original_pr.number), + status=IncidentStatus.RESOLVED.value, + creation_date=revert_pr_map.original_pr.state_changed_at, + acknowledged_date=revert_pr_map.revert_pr.created_at, + resolved_date=revert_pr_map.revert_pr.state_changed_at, + assigned_to=revert_pr_map.revert_pr.author, + assignees=[revert_pr_map.revert_pr.author], + meta={ + "revert_pr": self._adapt_pr_to_json(revert_pr_map.revert_pr), + "original_pr": self._adapt_pr_to_json(revert_pr_map.original_pr), + "created_at": revert_pr_map.revert_pr.created_at.isoformat(), + "updated_at": revert_pr_map.revert_pr.updated_at.isoformat(), + }, + created_at=existing_incident.created_at + if existing_incident + else time_now(), + updated_at=time_now(), + incident_type=IncidentType.REVERT_PR, + ) + incident_org_incident_service_map_model = IncidentOrgIncidentServiceMap( + incident_id=incident_id, + service_id=org_incident_service.id, + ) + + return incident, incident_org_incident_service_map_model + + @staticmethod + def _adapt_org_incident_service( + org_repo: OrgRepo, + org_incident_service: OrgIncidentService, + ) -> OrgIncidentService: + + return OrgIncidentService( + id=org_incident_service.id if org_incident_service else uuid4_str(), + org_id=org_repo.org_id, + provider=org_repo.provider, + name=org_repo.name, + key=str(org_repo.id), + meta={}, + created_at=org_incident_service.created_at + if org_incident_service + else time_now(), + updated_at=time_now(), + source_type=IncidentSource.GIT_REPO, + ) + + @staticmethod + def _adapt_pr_to_json(pr: PullRequest) -> Dict[str, any]: + return { + "id": str(pr.id), + "repo_id": str(pr.repo_id), + "number": pr.number, + "title": pr.title, + "state": pr.state.value, + "author": pr.author, + "reviewers": pr.reviewers or [], + "url": pr.url, + "base_branch": pr.base_branch, + "head_branch": pr.head_branch, + "state_changed_at": pr.state_changed_at.isoformat() + if pr.state_changed_at + else None, + "commits": pr.commits, + "comments": pr.comments, + "provider": pr.provider, + } + + +def get_incidents_sync_etl_handler(org_id: str) -> GitIncidentsETLHandler: + return GitIncidentsETLHandler( + org_id, + get_git_incidents_api_service(), + IncidentsRepoService(), + ) diff --git a/backend/analytics_server/dora/service/incidents/sync/etl_handler.py b/backend/analytics_server/dora/service/incidents/sync/etl_handler.py new file mode 100644 index 000000000..455a48c84 --- /dev/null +++ b/backend/analytics_server/dora/service/incidents/sync/etl_handler.py @@ -0,0 +1,107 @@ +from datetime import timedelta +from typing import List + +from dora.service.incidents.integration import get_incidents_integration_service +from dora.service.incidents.sync.etl_incidents_factory import IncidentsETLFactory +from dora.service.incidents.sync.etl_provider_handler import IncidentsProviderETLHandler +from dora.store.models.incidents import ( + OrgIncidentService, + IncidentBookmarkType, + IncidentProvider, + IncidentsBookmark, +) +from dora.store.repos.incidents import IncidentsRepoService +from dora.utils.log import LOG +from dora.utils.string import uuid4_str +from dora.utils.time import time_now + + +class IncidentsETLHandler: + def __init__( + self, + provider: IncidentProvider, + incident_repo_service: IncidentsRepoService, + etl_service: IncidentsProviderETLHandler, + ): + self.provider = provider + self.incident_repo_service = incident_repo_service + self.etl_service = etl_service + + def sync_org_incident_services(self, org_id: str): + try: + incident_services = self.incident_repo_service.get_org_incident_services( + org_id + ) + updated_services = self.etl_service.get_updated_incident_services( + incident_services + ) + self.incident_repo_service.update_org_incident_services(updated_services) + for service in updated_services: + try: + self._sync_service_incidents(service) + except Exception as e: + LOG.error( + f"Error syncing incidents for service {service.key}: {str(e)}" + ) + continue + except Exception as e: + LOG.error(f"Error syncing incident services for org {org_id}: {str(e)}") + return + + def _sync_service_incidents(self, service: OrgIncidentService): + try: + bookmark = self.__get_incidents_bookmark(service) + ( + incidents, + incident_org_incident_service_map, + bookmark, + ) = self.etl_service.process_service_incidents(service, bookmark) + self.incident_repo_service.save_incidents_data( + incidents, incident_org_incident_service_map + ) + self.incident_repo_service.save_incidents_bookmark(bookmark) + + except Exception as e: + LOG.error(f"Error syncing incidents for service {service.key}: {str(e)}") + return + + def __get_incidents_bookmark( + self, service: OrgIncidentService, default_sync_days: int = 31 + ): + bookmark = self.incident_repo_service.get_incidents_bookmark( + str(service.id), IncidentBookmarkType.SERVICE, self.provider + ) + if not bookmark: + default_pr_bookmark = time_now() - timedelta(days=default_sync_days) + bookmark = IncidentsBookmark( + id=uuid4_str(), + entity_id=str(service.id), + entity_type=IncidentBookmarkType.SERVICE, + provider=self.provider.value, + bookmark=default_pr_bookmark, + ) + return bookmark + + +def sync_org_incidents(org_id: str): + incident_providers: List[ + str + ] = get_incidents_integration_service().get_org_providers(org_id) + if not incident_providers: + LOG.info(f"No incident providers found for org {org_id}") + return + etl_factory = IncidentsETLFactory(org_id) + + for provider in incident_providers: + try: + incident_provider = IncidentProvider(provider) + incidents_etl_handler = IncidentsETLHandler( + incident_provider, IncidentsRepoService(), etl_factory(provider) + ) + incidents_etl_handler.sync_org_incident_services(org_id) + except Exception as e: + LOG.error( + f"Error syncing incidents for provider {provider}, org {org_id}: {str(e)}" + ) + continue + LOG.info(f"Synced incidents for org {org_id}") diff --git a/backend/analytics_server/dora/service/incidents/sync/etl_incidents_factory.py b/backend/analytics_server/dora/service/incidents/sync/etl_incidents_factory.py new file mode 100644 index 000000000..1d536bf0b --- /dev/null +++ b/backend/analytics_server/dora/service/incidents/sync/etl_incidents_factory.py @@ -0,0 +1,15 @@ +from dora.service.incidents.sync.etl_git_incidents_handler import ( + get_incidents_sync_etl_handler, +) +from dora.service.incidents.sync.etl_provider_handler import IncidentsProviderETLHandler +from dora.store.models.incidents import IncidentProvider + + +class IncidentsETLFactory: + def __init__(self, org_id: str): + self.org_id = org_id + + def __call__(self, provider: str) -> IncidentsProviderETLHandler: + if provider == IncidentProvider.GITHUB.value: + return get_incidents_sync_etl_handler(self.org_id) + raise NotImplementedError(f"Unknown provider - {provider}") diff --git a/backend/analytics_server/dora/service/incidents/sync/etl_provider_handler.py b/backend/analytics_server/dora/service/incidents/sync/etl_provider_handler.py new file mode 100644 index 000000000..9a8eeadae --- /dev/null +++ b/backend/analytics_server/dora/service/incidents/sync/etl_provider_handler.py @@ -0,0 +1,43 @@ +from abc import ABC, abstractmethod +from typing import List, Tuple + +from dora.store.models.incidents import ( + OrgIncidentService, + IncidentsBookmark, + Incident, + IncidentOrgIncidentServiceMap, +) + + +class IncidentsProviderETLHandler(ABC): + @abstractmethod + def check_pat_validity(self) -> bool: + """ + This method checks if the PAT is valid. + :return: True if PAT is valid, False otherwise + :raises: Exception if PAT is invalid + """ + pass + + @abstractmethod + def get_updated_incident_services( + self, incident_services: List[OrgIncidentService] + ) -> List[OrgIncidentService]: + """ + This method returns the updated incident services. + :param incident_services: List of incident services + :return: List of updated incident services + """ + pass + + @abstractmethod + def process_service_incidents( + self, incident_service: OrgIncidentService, bookmark: IncidentsBookmark + ) -> Tuple[List[Incident], List[IncidentOrgIncidentServiceMap], IncidentsBookmark]: + """ + This method processes the incidents for the incident services. + :param incident_service: Incident service object + :param bookmark: IncidentsBookmark object + :return: Tuple of incidents, incident service map and incidents bookmark + """ + pass diff --git a/backend/analytics_server/dora/service/merge_to_deploy_broker/__init__.py b/backend/analytics_server/dora/service/merge_to_deploy_broker/__init__.py new file mode 100644 index 000000000..9eca24263 --- /dev/null +++ b/backend/analytics_server/dora/service/merge_to_deploy_broker/__init__.py @@ -0,0 +1,2 @@ +from .mtd_handler import process_merge_to_deploy_cache +from .utils import get_merge_to_deploy_broker_utils_service, MergeToDeployBrokerUtils diff --git a/backend/analytics_server/dora/service/merge_to_deploy_broker/mtd_handler.py b/backend/analytics_server/dora/service/merge_to_deploy_broker/mtd_handler.py new file mode 100644 index 000000000..1196fe770 --- /dev/null +++ b/backend/analytics_server/dora/service/merge_to_deploy_broker/mtd_handler.py @@ -0,0 +1,126 @@ +from datetime import datetime +from typing import List + +from dora.service.deployments import DeploymentPRMapperService +from dora.store.models.code import ( + PullRequest, + OrgRepo, + RepoWorkflow, + BookmarkMergeToDeployBroker, + RepoWorkflowRuns, + RepoWorkflowRunsStatus, +) +from dora.store.repos.code import CodeRepoService +from dora.store.repos.workflows import WorkflowRepoService +from dora.utils.lock import RedisLockService, get_redis_lock_service + +DEPLOYMENTS_TO_PROCESS = 500 + + +class MergeToDeployCacheHandler: + def __init__( + self, + org_id: str, + code_repo_service: CodeRepoService, + workflow_repo_service: WorkflowRepoService, + deployment_pr_mapper_service: DeploymentPRMapperService, + redis_lock_service: RedisLockService, + ): + self.org_id = org_id + self.code_repo_service = code_repo_service + self.workflow_repo_service = workflow_repo_service + self.deployment_pr_mapper_service = deployment_pr_mapper_service + self.redis_lock_service = redis_lock_service + + def process_org_mtd(self): + org_repos: List[OrgRepo] = self.code_repo_service.get_active_org_repos( + self.org_id + ) + for org_repo in org_repos: + try: + with self.redis_lock_service.acquire_lock( + "{org_repo}:" + f"{str(org_repo.id)}:merge_to_deploy_broker" + ): + self._process_deployments_for_merge_to_deploy_caching( + str(org_repo.id) + ) + except Exception as e: + print(f"Error syncing workflow for repo {str(org_repo.id)}: {str(e)}") + continue + + def _process_deployments_for_merge_to_deploy_caching(self, repo_id: str): + org_repo: OrgRepo = self.code_repo_service.get_repo_by_id(repo_id) + if not org_repo: + Exception(f"Repo with {repo_id} not found") + + repo_workflows: List[ + RepoWorkflow + ] = self.workflow_repo_service.get_repo_workflows_by_repo_id(repo_id) + if not repo_workflows: + return + + broker_bookmark: BookmarkMergeToDeployBroker = ( + self.code_repo_service.get_merge_to_deploy_broker_bookmark(repo_id) + ) + if not broker_bookmark: + broker_bookmark = BookmarkMergeToDeployBroker(repo_id=repo_id) + + bookmark_time: datetime = broker_bookmark.bookmark_date + + repo_workflow_runs: List[ + RepoWorkflowRuns + ] = self.workflow_repo_service.get_repo_workflow_runs_conducted_after_time( + repo_id, bookmark_time, DEPLOYMENTS_TO_PROCESS + ) + + if not repo_workflow_runs: + return + + for repo_workflow_run in repo_workflow_runs: + try: + self.code_repo_service.get_merge_to_deploy_broker_bookmark(repo_id) + self._cache_prs_merge_to_deploy_for_repo_workflow_run( + repo_id, repo_workflow_run + ) + conducted_at: datetime = repo_workflow_run.conducted_at + broker_bookmark.bookmark = conducted_at.isoformat() + self.code_repo_service.update_merge_to_deploy_broker_bookmark( + broker_bookmark + ) + except Exception as e: + raise Exception(f"Error caching prs for repo {repo_id}: {str(e)}") + + def _cache_prs_merge_to_deploy_for_repo_workflow_run( + self, repo_id: str, repo_workflow_run: RepoWorkflowRuns + ): + if repo_workflow_run.status != RepoWorkflowRunsStatus.SUCCESS: + return + + conducted_at: datetime = repo_workflow_run.conducted_at + relevant_prs: List[ + PullRequest + ] = self.code_repo_service.get_prs_in_repo_merged_before_given_date_with_merge_to_deploy_as_null( + repo_id, conducted_at + ) + prs_to_update: List[ + PullRequest + ] = self.deployment_pr_mapper_service.get_all_prs_deployed( + relevant_prs, repo_workflow_run + ) + + for pr in prs_to_update: + pr.merge_to_deploy = int( + (conducted_at - pr.state_changed_at).total_seconds() + ) + self.code_repo_service.update_prs(prs_to_update) + + +def process_merge_to_deploy_cache(org_id: str): + merge_to_deploy_cache_handler = MergeToDeployCacheHandler( + org_id, + CodeRepoService(), + WorkflowRepoService(), + DeploymentPRMapperService(), + get_redis_lock_service(), + ) + merge_to_deploy_cache_handler.process_org_mtd() diff --git a/backend/analytics_server/dora/service/merge_to_deploy_broker/utils.py b/backend/analytics_server/dora/service/merge_to_deploy_broker/utils.py new file mode 100644 index 000000000..eefdf3fee --- /dev/null +++ b/backend/analytics_server/dora/service/merge_to_deploy_broker/utils.py @@ -0,0 +1,49 @@ +from datetime import datetime +from typing import List + +from dora.store.models.code import ( + PullRequest, + PullRequestState, + BookmarkMergeToDeployBroker, +) +from dora.store.repos.code import CodeRepoService +from dora.utils.lock import get_redis_lock_service, RedisLockService + + +class MergeToDeployBrokerUtils: + def __init__( + self, code_repo_service: CodeRepoService, redis_lock_service: RedisLockService + ): + self.code_repo_service = code_repo_service + self.redis_lock_service = redis_lock_service + + def pushback_merge_to_deploy_bookmark(self, repo_id: str, prs: List[PullRequest]): + with self.redis_lock_service.acquire_lock( + "{org_repo}:" + f"{repo_id}:merge_to_deploy_broker" + ): + self._pushback_merge_to_deploy_bookmark(repo_id, prs) + + def _pushback_merge_to_deploy_bookmark(self, repo_id: str, prs: List[PullRequest]): + merged_prs = [pr for pr in prs if pr.state == PullRequestState.MERGED] + if not merged_prs: + return + + min_merged_time: datetime = min([pr.state_changed_at for pr in merged_prs]) + + merge_to_deploy_broker_bookmark: BookmarkMergeToDeployBroker = ( + self.code_repo_service.get_merge_to_deploy_broker_bookmark(repo_id) + ) + if not merge_to_deploy_broker_bookmark: + merge_to_deploy_broker_bookmark = BookmarkMergeToDeployBroker( + repo_id=repo_id, bookmark=min_merged_time.isoformat() + ) + + self.code_repo_service.update_merge_to_deploy_broker_bookmark( + merge_to_deploy_broker_bookmark + ) + + +def get_merge_to_deploy_broker_utils_service(): + return MergeToDeployBrokerUtils( + CodeRepoService(), redis_lock_service=get_redis_lock_service() + ) diff --git a/backend/analytics_server/dora/service/pr_analytics.py b/backend/analytics_server/dora/service/pr_analytics.py new file mode 100644 index 000000000..f540e3fa9 --- /dev/null +++ b/backend/analytics_server/dora/service/pr_analytics.py @@ -0,0 +1,22 @@ +from dora.store.models.code import OrgRepo, PullRequest +from dora.store.repos.code import CodeRepoService + +from typing import List + + +class PullRequestAnalyticsService: + def __init__(self, code_repo_service: CodeRepoService): + self.code_repo_service: CodeRepoService = code_repo_service + + def get_prs_by_ids(self, pr_ids: List[str]) -> List[PullRequest]: + return self.code_repo_service.get_prs_by_ids(pr_ids) + + def get_team_repos(self, team_id: str) -> List[OrgRepo]: + return self.code_repo_service.get_team_repos(team_id) + + def get_repo_by_id(self, team_id: str) -> List[OrgRepo]: + return self.code_repo_service.get_repo_by_id(team_id) + + +def get_pr_analytics_service(): + return PullRequestAnalyticsService(CodeRepoService()) diff --git a/backend/analytics_server/dora/service/query_validator.py b/backend/analytics_server/dora/service/query_validator.py new file mode 100644 index 000000000..1f6834bc6 --- /dev/null +++ b/backend/analytics_server/dora/service/query_validator.py @@ -0,0 +1,77 @@ +from datetime import timedelta +from typing import List + +from werkzeug.exceptions import NotFound, BadRequest + +from dora.store.models.core import Organization, Team, Users +from dora.store.repos.core import CoreRepoService +from dora.utils.time import Interval + +DEFAULT_ORG_NAME = "default" + + +class QueryValidator: + def __init__(self, repo_service: CoreRepoService): + self.repo_service = repo_service + + def get_default_org(self) -> Organization: + org: Organization = self.repo_service.get_org_by_name(DEFAULT_ORG_NAME) + if org is None: + raise NotFound("Default org not found") + return org + + def org_validator(self, org_id: str) -> Organization: + org: Organization = self.repo_service.get_org(org_id) + if org is None: + raise NotFound(f"Org {org_id} not found") + return org + + def team_validator(self, team_id: str) -> Team: + team: Team = self.repo_service.get_team(team_id) + if team is None: + raise NotFound(f"Team {team_id} not found") + return team + + def teams_validator(self, team_ids: List[str]) -> List[Team]: + teams: List[Team] = self.repo_service.get_teams(team_ids) + if len(teams) != len(team_ids): + query_team_ids = set(team_ids) + found_team_ids = set(map(lambda x: str(x.id), teams)) + missing_team_ids = query_team_ids - found_team_ids + raise NotFound(f"Team(s) not found: {missing_team_ids}") + return teams + + def interval_validator( + self, from_time, to_time, interval_limit_in_days: int = 105 + ) -> Interval: + if None in (from_time.tzinfo, to_time.tzinfo): + raise BadRequest("Timestamp passed without tz info") + interval = Interval(from_time, to_time) + if interval_limit_in_days is not None and interval.duration > timedelta( + days=interval_limit_in_days + ): + raise BadRequest( + f"Only {interval_limit_in_days} days duration is supported" + ) + return interval + + def user_validator(self, user_id: str) -> Users: + user = self.repo_service.get_user(user_id) + if user is None: + raise NotFound(f"User {user_id} not found") + return user + + def users_validator(self, user_ids: List[str]) -> List[Users]: + users: List[Users] = self.repo_service.get_users(user_ids) + + if len(users) != len(user_ids): + query_user_ids = set(user_ids) + found_user_ids = set(map(lambda x: str(x.id), users)) + missing_user_ids = query_user_ids - found_user_ids + raise NotFound(f"User(s) not found: {missing_user_ids}") + + return users + + +def get_query_validator(): + return QueryValidator(CoreRepoService()) diff --git a/backend/analytics_server/dora/service/settings/__init__.py b/backend/analytics_server/dora/service/settings/__init__.py new file mode 100644 index 000000000..aca812775 --- /dev/null +++ b/backend/analytics_server/dora/service/settings/__init__.py @@ -0,0 +1,2 @@ +from .configuration_settings import SettingsService, get_settings_service +from .setting_type_validator import settings_type_validator diff --git a/backend/analytics_server/dora/service/settings/configuration_settings.py b/backend/analytics_server/dora/service/settings/configuration_settings.py new file mode 100644 index 000000000..4dc08c6da --- /dev/null +++ b/backend/analytics_server/dora/service/settings/configuration_settings.py @@ -0,0 +1,376 @@ +from typing import Any, Dict, Optional, List + +from dora.service.settings.default_settings_data import get_default_setting_data +from dora.service.settings.models import ( + ConfigurationSettings, + ExcludedPRsSetting, + IncidentSettings, + IncidentSourcesSetting, + IncidentTypesSetting, +) +from dora.store.models.core.users import Users +from dora.store.models.incidents import IncidentSource, IncidentType +from dora.store.models.settings import SettingType, Settings, EntityType +from dora.store.repos.settings import SettingsRepoService +from dora.utils.time import time_now + + +class SettingsService: + def __init__(self, _settings_repo): + self._settings_repo: SettingsRepoService = _settings_repo + + def _adapt_specific_incident_setting_from_setting_data(self, data: Dict[str, any]): + """ + Adapts the json data in Settings.data to IncidentSettings + """ + + return IncidentSettings(title_filters=data.get("title_filters", [])) + + def _adapt_excluded_prs_setting_from_setting_data(self, data: Dict[str, any]): + """ + Adapts the json data in Setting.data for SettingType EXCLUDED_PRS_SETTING to ExcludedPRsSetting + """ + return ExcludedPRsSetting(excluded_pr_ids=data.get("excluded_pr_ids", [])) + + def _adapt_incident_source_setting_from_setting_data( + self, data: Dict[str, any] + ) -> IncidentSourcesSetting: + """ + Adapts the json data in Settings.data to IncidentSourcesSetting + """ + return IncidentSourcesSetting( + incident_sources=[ + IncidentSource(source) for source in data.get("incident_sources") or [] + ] + ) + + def _adapt_incident_types_setting_from_setting_data( + self, data: Dict[str, any] + ) -> IncidentTypesSetting: + """ + Adapts the json data in Settings.data to IncidentTypesSetting + """ + + return IncidentTypesSetting( + incident_types=[ + IncidentType(incident_type) + for incident_type in data.get("incident_types") or [] + ] + ) + + def _handle_config_setting_from_db_setting( + self, setting_type: SettingType, setting_data + ): + # Add if statements and adapters for new setting types + + if setting_type == SettingType.INCIDENT_SETTING: + return self._adapt_specific_incident_setting_from_setting_data(setting_data) + + if setting_type == SettingType.EXCLUDED_PRS_SETTING: + return self._adapt_excluded_prs_setting_from_setting_data(setting_data) + + if setting_type == SettingType.INCIDENT_TYPES_SETTING: + return self._adapt_incident_types_setting_from_setting_data(setting_data) + + if setting_type == SettingType.INCIDENT_SOURCES_SETTING: + return self._adapt_incident_source_setting_from_setting_data(setting_data) + + raise Exception(f"Invalid Setting Type: {setting_type}") + + def _adapt_config_setting_from_db_setting(self, setting: Settings): + specific_setting = self._handle_config_setting_from_db_setting( + setting.setting_type, setting.data + ) + + return ConfigurationSettings( + entity_id=setting.entity_id, + entity_type=setting.entity_type, + updated_by=setting.updated_by, + created_at=setting.created_at, + updated_at=setting.updated_at, + specific_settings=specific_setting, + ) + + def get_settings( + self, setting_type: SettingType, entity_type: EntityType, entity_id: str + ) -> Optional[ConfigurationSettings]: + + setting = self._settings_repo.get_setting( + entity_id=entity_id, + entity_type=entity_type, + setting_type=setting_type, + ) + if not setting: + return None + + return self._adapt_config_setting_from_db_setting(setting) + + def get_or_set_settings_for_multiple_entity_ids( + self, + setting_type: SettingType, + entity_type: EntityType, + entity_ids: List[str], + setter: Users = None, + ) -> List[ConfigurationSettings]: + + settings = self._settings_repo.get_settings_for_multiple_entity_ids( + entity_ids, entity_type, setting_type + ) + + current_entity_ids = set([str(setting.entity_id) for setting in settings]) + missing_entity_ids = set(entity_ids).difference(current_entity_ids) + if missing_entity_ids: + data = get_default_setting_data(setting_type) + settings_to_create = [ + Settings( + entity_id=entity_id, + entity_type=entity_type, + setting_type=setting_type, + updated_by=setter.id if setter else None, + data=data, + created_at=time_now(), + updated_at=time_now(), + is_deleted=False, + ) + for entity_id in missing_entity_ids + ] + new_settings = self._settings_repo.create_settings(settings_to_create) + settings.extend(new_settings) + + return list(map(self._adapt_config_setting_from_db_setting, settings)) + + def _adapt_specific_incident_setting_from_json( + self, data: Dict[str, any] + ) -> IncidentSettings: + """ + Adapts the json data from API to IncidentSettings + """ + + return IncidentSettings(title_filters=data.get("title_includes", [])) + + def _adapt_excluded_prs_setting_from_json(self, data: Dict[str, any]): + """ + Adapts the json data from API for SettingType EXCLUDED_PRS_SETTING to ExcludedPrsSetting + """ + return ExcludedPRsSetting(excluded_pr_ids=data.get("excluded_pr_ids", [])) + + def _adapt_incident_source_setting_from_json( + self, data: Dict[str, any] + ) -> IncidentSourcesSetting: + """ + Adapts the json data from API to IncidentSourcesSetting + """ + + return IncidentSourcesSetting( + incident_sources=[ + IncidentSource(source) for source in data.get("incident_sources") or [] + ] + ) + + def _adapt_incident_types_setting_from_json( + self, data: Dict[str, any] + ) -> IncidentTypesSetting: + """ + Adapts the json data from API to IncidentTypesSetting + """ + + return IncidentTypesSetting( + incident_types=[ + IncidentType(incident_type) + for incident_type in data.get("incident_types") or [] + ] + ) + + def _handle_config_setting_from_json_data( + self, setting_type: SettingType, setting_data + ): + # Add if statements and adapters for new setting types + + if setting_type == SettingType.INCIDENT_SETTING: + return self._adapt_specific_incident_setting_from_json(setting_data) + + if setting_type == SettingType.EXCLUDED_PRS_SETTING: + return self._adapt_excluded_prs_setting_from_json(setting_data) + + if setting_type == SettingType.INCIDENT_SOURCES_SETTING: + return self._adapt_incident_source_setting_from_json(setting_data) + + if setting_type == SettingType.INCIDENT_TYPES_SETTING: + return self._adapt_incident_types_setting_from_json(setting_data) + + raise Exception(f"Invalid Setting Type: {setting_type}") + + def _adapt_incident_setting_json_data( + self, + specific_setting: IncidentSettings, + ): + return {"title_filters": specific_setting.title_filters} + + def _adapt_excluded_prs_setting_json_data( + self, specific_setting: ExcludedPRsSetting + ): + return {"excluded_pr_ids": specific_setting.excluded_pr_ids} + + def _adapt_incident_source_setting_json_data( + self, specific_setting: IncidentSourcesSetting + ) -> Dict: + return { + "incident_sources": [ + source.value for source in specific_setting.incident_sources + ] + } + + def _adapt_incident_types_setting_json_data( + self, specific_setting: IncidentTypesSetting + ) -> Dict: + return { + "incident_types": [ + incident_type.value for incident_type in specific_setting.incident_types + ] + } + + def _handle_config_setting_to_db_setting( + self, setting_type: SettingType, specific_setting + ): + # Add if statements and adapters to get data for new setting types + + if setting_type == SettingType.INCIDENT_SETTING and isinstance( + specific_setting, IncidentSettings + ): + return self._adapt_incident_setting_json_data(specific_setting) + if setting_type == SettingType.EXCLUDED_PRS_SETTING and isinstance( + specific_setting, ExcludedPRsSetting + ): + return self._adapt_excluded_prs_setting_json_data(specific_setting) + + if setting_type == SettingType.INCIDENT_TYPES_SETTING and isinstance( + specific_setting, IncidentTypesSetting + ): + return self._adapt_incident_types_setting_json_data(specific_setting) + + if setting_type == SettingType.INCIDENT_SOURCES_SETTING and isinstance( + specific_setting, IncidentSourcesSetting + ): + return self._adapt_incident_source_setting_json_data(specific_setting) + + raise Exception(f"Invalid Setting Type: {setting_type}") + + def _adapt_specific_setting_data_from_json( + self, setting_type: SettingType, setting_data: dict + ): + """ + This function is getting json data (setting_data) and adapting it to the data class as per the setting type. + This then again converts the class data into a dictionary and returns it. + + The process has been done in order to just maintain the data sanctity and to avoid any un-formatted data being stored in the DB. + """ + + specific_setting = self._handle_config_setting_from_json_data( + setting_type, setting_data + ) + + return self._handle_config_setting_to_db_setting(setting_type, specific_setting) + + def save_settings( + self, + setting_type: SettingType, + entity_type: EntityType, + entity_id: str, + setter: Users = None, + setting_data: Dict = None, + ) -> ConfigurationSettings: + + if setting_data: + data = self._adapt_specific_setting_data_from_json( + setting_type, setting_data + ) + else: + data = get_default_setting_data(setting_type) + + setting = Settings( + entity_id=entity_id, + entity_type=entity_type, + setting_type=setting_type, + updated_by=setter.id if setter else None, + data=data, + created_at=time_now(), + updated_at=time_now(), + is_deleted=False, + ) + + saved_setting = self._settings_repo.save_setting(setting) + + return self._adapt_config_setting_from_db_setting(saved_setting) + + def delete_settings( + self, + setting_type: SettingType, + entity_type: EntityType, + deleted_by: Users, + entity_id: str, + ) -> ConfigurationSettings: + + return self._adapt_config_setting_from_db_setting( + self._settings_repo.delete_setting( + setting_type=setting_type, + entity_id=entity_id, + entity_type=entity_type, + deleted_by=deleted_by, + ) + ) + + def get_settings_map( + self, + entity_id: str, + setting_types: List[SettingType], + entity_type: EntityType, + ignore_default_setting_type: List[SettingType] = None, + ) -> Dict[SettingType, any]: + + if not ignore_default_setting_type: + ignore_default_setting_type = [] + + settings: List[Settings] = self._settings_repo.get_settings( + entity_id=entity_id, setting_types=setting_types, entity_type=entity_type + ) + setting_type_to_setting_map: Dict[ + SettingType, Any + ] = self._get_setting_type_to_setting_map( + setting_types, settings, ignore_default_setting_type + ) + + return setting_type_to_setting_map + + def _get_setting_type_to_setting_map( + self, + setting_types: List[SettingType], + settings: List[Settings], + ignore_default_setting_type: List[SettingType] = None, + ) -> Dict[SettingType, Any]: + + if not ignore_default_setting_type: + ignore_default_setting_type = [] + + setting_type_to_setting_map: Dict[SettingType, Any] = {} + for setting in settings: + setting_type_to_setting_map[ + setting.setting_type + ] = self._adapt_config_setting_from_db_setting(setting).specific_settings + + for setting_type in setting_types: + if (setting_type not in setting_type_to_setting_map) and ( + setting_type not in ignore_default_setting_type + ): + setting_type_to_setting_map[setting_type] = self.get_default_setting( + setting_type + ) + return setting_type_to_setting_map + + def get_default_setting(self, setting_type: SettingType): + return self._handle_config_setting_from_db_setting( + setting_type, get_default_setting_data(setting_type) + ) + + +def get_settings_service(): + return SettingsService(SettingsRepoService()) diff --git a/backend/analytics_server/dora/service/settings/default_settings_data.py b/backend/analytics_server/dora/service/settings/default_settings_data.py new file mode 100644 index 000000000..c9ec1c0e9 --- /dev/null +++ b/backend/analytics_server/dora/service/settings/default_settings_data.py @@ -0,0 +1,29 @@ +from dora.store.models.incidents import IncidentSource, IncidentType +from dora.store.models.settings import SettingType + + +MIN_CYCLE_TIME_THRESHOLD = 3600 + + +def get_default_setting_data(setting_type: SettingType): + if setting_type == SettingType.INCIDENT_SETTING: + return {"title_filters": []} + + if setting_type == SettingType.EXCLUDED_PRS_SETTING: + return {"excluded_pr_ids": []} + + if setting_type == SettingType.INCIDENT_SOURCES_SETTING: + incident_sources = list(IncidentSource) + return { + "incident_sources": [ + incident_source.value for incident_source in incident_sources + ] + } + + if setting_type == SettingType.INCIDENT_TYPES_SETTING: + incident_types = list(IncidentType) + return { + "incident_types": [incident_type.value for incident_type in incident_types] + } + + raise Exception(f"Invalid Setting Type: {setting_type}") diff --git a/backend/analytics_server/dora/service/settings/models.py b/backend/analytics_server/dora/service/settings/models.py new file mode 100644 index 000000000..0bbfff43c --- /dev/null +++ b/backend/analytics_server/dora/service/settings/models.py @@ -0,0 +1,41 @@ +from dataclasses import dataclass +from datetime import datetime +from typing import List + +from dora.store.models import EntityType +from dora.store.models.incidents.enums import IncidentSource, IncidentType + + +@dataclass +class BaseSetting: + pass + + +@dataclass +class ConfigurationSettings: + entity_id: str + entity_type: EntityType + specific_settings: BaseSetting + updated_by: str + created_at: datetime + updated_at: datetime + + +@dataclass +class IncidentSettings(BaseSetting): + title_filters: List[str] + + +@dataclass +class ExcludedPRsSetting(BaseSetting): + excluded_pr_ids: List[str] + + +@dataclass +class IncidentTypesSetting(BaseSetting): + incident_types: List[IncidentType] + + +@dataclass +class IncidentSourcesSetting(BaseSetting): + incident_sources: List[IncidentSource] diff --git a/backend/analytics_server/dora/service/settings/setting_type_validator.py b/backend/analytics_server/dora/service/settings/setting_type_validator.py new file mode 100644 index 000000000..1e8415de2 --- /dev/null +++ b/backend/analytics_server/dora/service/settings/setting_type_validator.py @@ -0,0 +1,19 @@ +from werkzeug.exceptions import BadRequest + +from dora.store.models.settings import SettingType + + +def settings_type_validator(setting_type: str): + if setting_type == SettingType.INCIDENT_SETTING.value: + return SettingType.INCIDENT_SETTING + + if setting_type == SettingType.EXCLUDED_PRS_SETTING.value: + return SettingType.EXCLUDED_PRS_SETTING + + if setting_type == SettingType.INCIDENT_TYPES_SETTING.value: + return SettingType.INCIDENT_TYPES_SETTING + + if setting_type == SettingType.INCIDENT_SOURCES_SETTING.value: + return SettingType.INCIDENT_SOURCES_SETTING + + raise BadRequest(f"Invalid Setting Type: {setting_type}") diff --git a/backend/analytics_server/dora/service/sync_data.py b/backend/analytics_server/dora/service/sync_data.py new file mode 100644 index 000000000..d5accbfa5 --- /dev/null +++ b/backend/analytics_server/dora/service/sync_data.py @@ -0,0 +1,34 @@ +from dora.service.code import sync_code_repos +from dora.service.incidents import sync_org_incidents +from dora.service.merge_to_deploy_broker import process_merge_to_deploy_cache +from dora.service.query_validator import get_query_validator +from dora.service.workflows import sync_org_workflows +from dora.utils.log import LOG + +sync_sequence = [ + sync_code_repos, + sync_org_workflows, + process_merge_to_deploy_cache, + sync_org_incidents, +] + + +def trigger_data_sync(): + default_org = get_query_validator().get_default_org() + org_id = str(default_org.id) + LOG.info(f"Starting data sync for org {org_id}") + + for sync_func in sync_sequence: + try: + sync_func(org_id) + LOG.info(f"Data sync for {sync_func.__name__} completed successfully") + except Exception as e: + LOG.error( + f"Error syncing {sync_func.__name__} data for org {org_id}: {str(e)}" + ) + continue + LOG.info(f"Data sync for org {org_id} completed successfully") + + +if __name__ == "__main__": + trigger_data_sync() diff --git a/backend/analytics_server/dora/service/workflows/__init__.py b/backend/analytics_server/dora/service/workflows/__init__.py new file mode 100644 index 000000000..c4ede6824 --- /dev/null +++ b/backend/analytics_server/dora/service/workflows/__init__.py @@ -0,0 +1 @@ +from .sync import sync_org_workflows diff --git a/backend/analytics_server/dora/service/workflows/integration.py b/backend/analytics_server/dora/service/workflows/integration.py new file mode 100644 index 000000000..6f4df4f9d --- /dev/null +++ b/backend/analytics_server/dora/service/workflows/integration.py @@ -0,0 +1,28 @@ +from typing import List + +from dora.store.models import Integration +from dora.store.models.code import RepoWorkflowProviders +from dora.store.repos.core import CoreRepoService + +WORKFLOW_INTEGRATION_BUCKET = [ + RepoWorkflowProviders.GITHUB_ACTIONS.value, +] + + +class WorkflowsIntegrationsService: + def __init__(self, core_repo_service: CoreRepoService): + self.core_repo_service = core_repo_service + + def get_org_providers(self, org_id: str) -> List[str]: + integrations: List[ + Integration + ] = self.core_repo_service.get_org_integrations_for_names( + org_id, WORKFLOW_INTEGRATION_BUCKET + ) + if not integrations: + return [] + return [integration.name for integration in integrations] + + +def get_workflows_integrations_service() -> WorkflowsIntegrationsService: + return WorkflowsIntegrationsService(CoreRepoService()) diff --git a/backend/analytics_server/dora/service/workflows/sync/__init__.py b/backend/analytics_server/dora/service/workflows/sync/__init__.py new file mode 100644 index 000000000..3a36308d5 --- /dev/null +++ b/backend/analytics_server/dora/service/workflows/sync/__init__.py @@ -0,0 +1 @@ +from .etl_handler import sync_org_workflows diff --git a/backend/analytics_server/dora/service/workflows/sync/etl_github_actions_handler.py b/backend/analytics_server/dora/service/workflows/sync/etl_github_actions_handler.py new file mode 100644 index 000000000..2ce3bd431 --- /dev/null +++ b/backend/analytics_server/dora/service/workflows/sync/etl_github_actions_handler.py @@ -0,0 +1,189 @@ +from datetime import datetime +from typing import Dict, Optional, List, Tuple +from uuid import uuid4 + +import pytz + +from dora.exapi.github import GithubApiService +from dora.service.workflows.sync.etl_provider_handler import WorkflowProviderETLHandler +from dora.store.models import UserIdentityProvider +from dora.store.models.code import ( + RepoWorkflowProviders, + RepoWorkflowRunsStatus, + RepoWorkflowRuns, + OrgRepo, + RepoWorkflowRunsBookmark, + RepoWorkflow, +) +from dora.store.repos.core import CoreRepoService +from dora.store.repos.workflows import WorkflowRepoService +from dora.utils.log import LOG +from dora.utils.time import ISO_8601_DATE_FORMAT, time_now + +DEFAULT_WORKFLOW_SYNC_DAYS = 31 +WORKFLOW_PROCESSING_CHUNK_SIZE = 100 + + +class GithubActionsETLHandler(WorkflowProviderETLHandler): + def __init__( + self, + org_id: str, + github_api_service: GithubApiService, + workflow_repo_service: WorkflowRepoService, + ): + self.org_id = org_id + self._api: GithubApiService = github_api_service + self._workflow_repo_service = workflow_repo_service + self._provider = RepoWorkflowProviders.GITHUB_ACTIONS.value + + def check_pat_validity(self) -> bool: + """ + This method checks if the PAT is valid. + :returns: PAT details + :raises: Exception if PAT is invalid + """ + is_valid = self._api.check_pat() + if not is_valid: + raise Exception("Github Personal Access Token is invalid") + return is_valid + + def get_workflow_runs( + self, + org_repo: OrgRepo, + repo_workflow: RepoWorkflow, + bookmark: RepoWorkflowRunsBookmark, + ) -> Tuple[List[RepoWorkflowRuns], RepoWorkflowRunsBookmark]: + """ + This method returns all workflow runs of a repo's workflow. After the bookmark date. + :param org_repo: OrgRepo object to get workflow runs for + :param repo_workflow: RepoWorkflow object to get workflow runs for + :param bookmark: Bookmark object to get all workflow runs after this date + :return: Workflow runs, Bookmark object + """ + bookmark_time_stamp = datetime.fromisoformat(bookmark.bookmark) + try: + github_workflow_runs = self._api.get_workflow_runs( + org_repo.org_name, + org_repo.name, + repo_workflow.provider_workflow_id, + bookmark_time_stamp, + ) + except Exception as e: + raise Exception( + f"[GitHub Sync Repo Workflow Worker] Error fetching workflow {str(repo_workflow.id)} " + f"for repo {str(org_repo.repo_id)}: {str(e)}" + ) + + if not github_workflow_runs: + LOG.info( + f"[GitHub Sync Repo Workflow Worker] No Workflow Runs found for " + f"Workflow: {str(repo_workflow.provider_workflow_id)}. Repo: {org_repo.org_name}/{org_repo.name}. " + f"Org: {self.org_id}" + ) + return [], bookmark + + bookmark.bookmark = self._get_new_bookmark_time_stamp( + github_workflow_runs + ).isoformat() + + repo_workflow_runs = [ + self._adapt_github_workflows_to_workflow_runs( + str(repo_workflow.id), workflow_run + ) + for workflow_run in github_workflow_runs + ] + + return repo_workflow_runs, bookmark + + def _get_new_bookmark_time_stamp( + self, github_workflow_runs: List[Dict] + ) -> datetime: + """ + This method returns the new bookmark timestamp for the workflow runs. + It returns the minimum timestamp of the pending jobs if there are any pending jobs. + This is done because there might be a workflow run that is still pending, and we + want to fetch it in the next sync. + """ + pending_job_timestamps = [ + self._get_datetime_from_gh_datetime(workflow_run["created_at"]) + for workflow_run in github_workflow_runs + if workflow_run["status"] != "completed" + ] + return min(pending_job_timestamps) if pending_job_timestamps else time_now() + + def _adapt_github_workflows_to_workflow_runs( + self, repo_workflow_id: str, github_workflow_run: Dict + ) -> RepoWorkflowRuns: + repo_workflow_run_in_db = self._workflow_repo_service.get_repo_workflow_run_by_provider_workflow_run_id( + repo_workflow_id, str(github_workflow_run["id"]) + ) + if repo_workflow_run_in_db: + workflow_run_id = repo_workflow_run_in_db.id + else: + workflow_run_id = uuid4() + return RepoWorkflowRuns( + id=workflow_run_id, + repo_workflow_id=repo_workflow_id, + provider_workflow_run_id=str(github_workflow_run["id"]), + event_actor=github_workflow_run["actor"]["login"], + head_branch=github_workflow_run["head_branch"], + status=self._get_repo_workflow_status(github_workflow_run), + created_at=time_now(), + updated_at=time_now(), + conducted_at=self._get_datetime_from_gh_datetime( + github_workflow_run["run_started_at"] + ), + duration=self._get_repo_workflow_run_duration(github_workflow_run), + meta=github_workflow_run, + html_url=github_workflow_run["html_url"], + ) + + @staticmethod + def _get_repo_workflow_status(github_workflow: Dict) -> RepoWorkflowRunsStatus: + if github_workflow["status"] != "completed": + return RepoWorkflowRunsStatus.PENDING + if github_workflow["conclusion"] == "success": + return RepoWorkflowRunsStatus.SUCCESS + return RepoWorkflowRunsStatus.FAILURE + + def _get_repo_workflow_run_duration( + self, github_workflow_run: Dict + ) -> Optional[int]: + + if not ( + github_workflow_run.get("updated_at") + and github_workflow_run.get("run_started_at") + ): + return None + + workflow_run_updated_at = self._get_datetime_from_gh_datetime( + github_workflow_run.get("updated_at") + ) + workflow_run_conducted_at = self._get_datetime_from_gh_datetime( + github_workflow_run.get("run_started_at") + ) + return int( + (workflow_run_updated_at - workflow_run_conducted_at).total_seconds() + ) + + @staticmethod + def _get_datetime_from_gh_datetime(datetime_str: str) -> datetime: + dt_without_timezone = datetime.strptime(datetime_str, ISO_8601_DATE_FORMAT) + return dt_without_timezone.replace(tzinfo=pytz.UTC) + + +def get_github_actions_etl_handler(org_id): + def _get_access_token(): + core_repo_service = CoreRepoService() + access_token = core_repo_service.get_access_token( + org_id, UserIdentityProvider.GITHUB + ) + if not access_token: + raise Exception( + f"Access token not found for org {org_id} and provider {UserIdentityProvider.GITHUB.value}" + ) + return access_token + + return GithubActionsETLHandler( + org_id, GithubApiService(_get_access_token()), WorkflowRepoService() + ) diff --git a/backend/analytics_server/dora/service/workflows/sync/etl_handler.py b/backend/analytics_server/dora/service/workflows/sync/etl_handler.py new file mode 100644 index 000000000..a813778bd --- /dev/null +++ b/backend/analytics_server/dora/service/workflows/sync/etl_handler.py @@ -0,0 +1,134 @@ +from datetime import timedelta +from typing import List, Tuple +from uuid import uuid4 + +from dora.service.code import get_code_integration_service +from dora.service.workflows.integration import get_workflows_integrations_service +from dora.service.workflows.sync.etl_provider_handler import WorkflowProviderETLHandler +from dora.service.workflows.sync.etl_workflows_factory import WorkflowETLFactory +from dora.store.models.code import ( + OrgRepo, + RepoWorkflow, + RepoWorkflowRunsBookmark, + RepoWorkflowRuns, + RepoWorkflowProviders, +) +from dora.store.repos.code import CodeRepoService +from dora.store.repos.workflows import WorkflowRepoService +from dora.utils.log import LOG +from dora.utils.time import time_now + + +class WorkflowETLHandler: + def __init__( + self, + code_repo_service: CodeRepoService, + workflow_repo_service: WorkflowRepoService, + etl_factory: WorkflowETLFactory, + ): + self.code_repo_service = code_repo_service + self.workflow_repo_service = workflow_repo_service + self.etl_factory = etl_factory + + def sync_org_workflows(self, org_id: str): + active_repo_workflows: List[ + Tuple[OrgRepo, RepoWorkflow] + ] = self._get_active_repo_workflows(org_id) + + for org_repo, repo_workflow in active_repo_workflows: + try: + self._sync_repo_workflow(org_repo, repo_workflow) + except Exception as e: + LOG.error( + f"Error syncing workflow for repo {repo_workflow.org_repo_id}: {str(e)}" + ) + continue + + def _get_active_repo_workflows( + self, org_id: str + ) -> List[Tuple[OrgRepo, RepoWorkflow]]: + code_providers: List[str] = get_code_integration_service().get_org_providers( + org_id + ) + workflow_providers: List[ + str + ] = get_workflows_integrations_service().get_org_providers(org_id) + if not code_providers or not workflow_providers: + LOG.info(f"No workflow integrations found for org {org_id}") + return [] + + org_repos: List[OrgRepo] = self.code_repo_service.get_active_org_repos(org_id) + repo_ids = [str(repo.id) for repo in org_repos] + repo_id_org_repo_map = {str(repo.id): repo for repo in org_repos} + active_repo_workflows: List[ + RepoWorkflow + ] = self.workflow_repo_service.get_active_repo_workflows_by_repo_ids_and_providers( + repo_ids, + [RepoWorkflowProviders(provider) for provider in workflow_providers], + ) + org_repo_workflows: List[Tuple[OrgRepo, RepoWorkflow]] = [] + for repo_workflow in active_repo_workflows: + org_repo_workflows.append( + (repo_id_org_repo_map[str(repo_workflow.org_repo_id)], repo_workflow) + ) + return org_repo_workflows + + def _sync_repo_workflow(self, org_repo: OrgRepo, repo_workflow: RepoWorkflow): + workflow_provider: RepoWorkflowProviders = repo_workflow.provider + etl_service: WorkflowProviderETLHandler = self.etl_factory( + workflow_provider.name + ) + if not etl_service.check_pat_validity(): + LOG.error("Invalid PAT for code provider") + return + try: + bookmark: RepoWorkflowRunsBookmark = self.__get_repo_workflow_bookmark( + repo_workflow + ) + repo_workflow_runs: List[RepoWorkflowRuns] + repo_workflow_runs, bookmark = etl_service.get_workflow_runs( + org_repo, repo_workflow, bookmark + ) + self.workflow_repo_service.save_repo_workflow_runs(repo_workflow_runs) + self.workflow_repo_service.update_repo_workflow_runs_bookmark(bookmark) + except Exception as e: + LOG.error( + f"Error syncing workflow for repo {repo_workflow.org_repo_id}: {str(e)}" + ) + return + + def __get_repo_workflow_bookmark( + self, repo_workflow: RepoWorkflow, default_sync_days: int = 31 + ) -> RepoWorkflowRunsBookmark: + repo_workflow_bookmark = ( + self.workflow_repo_service.get_repo_workflow_runs_bookmark(repo_workflow.id) + ) + if not repo_workflow_bookmark: + bookmark_string = ( + time_now() - timedelta(days=default_sync_days) + ).isoformat() + + repo_workflow_bookmark = RepoWorkflowRunsBookmark( + id=uuid4(), + repo_workflow_id=repo_workflow.id, + bookmark=bookmark_string, + created_at=time_now(), + updated_at=time_now(), + ) + return repo_workflow_bookmark + + +def sync_org_workflows(org_id: str): + workflow_providers: List[ + str + ] = get_workflows_integrations_service().get_org_providers(org_id) + if not workflow_providers: + LOG.info(f"No workflow integrations found for org {org_id}") + return + code_repo_service = CodeRepoService() + workflow_repo_service = WorkflowRepoService() + etl_factory = WorkflowETLFactory(org_id) + workflow_etl_handler = WorkflowETLHandler( + code_repo_service, workflow_repo_service, etl_factory + ) + workflow_etl_handler.sync_org_workflows(org_id) diff --git a/backend/analytics_server/dora/service/workflows/sync/etl_provider_handler.py b/backend/analytics_server/dora/service/workflows/sync/etl_provider_handler.py new file mode 100644 index 000000000..72b56aece --- /dev/null +++ b/backend/analytics_server/dora/service/workflows/sync/etl_provider_handler.py @@ -0,0 +1,36 @@ +from abc import ABC, abstractmethod +from typing import List, Tuple + +from dora.store.models.code import ( + OrgRepo, + RepoWorkflow, + RepoWorkflowRunsBookmark, + RepoWorkflowRuns, +) + + +class WorkflowProviderETLHandler(ABC): + @abstractmethod + def check_pat_validity(self) -> bool: + """ + This method checks if the PAT is valid. + :return: PAT details + :raises: Exception if PAT is invalid + """ + pass + + @abstractmethod + def get_workflow_runs( + self, + org_repo: OrgRepo, + repo_workflow: RepoWorkflow, + bookmark: RepoWorkflowRunsBookmark, + ) -> Tuple[List[RepoWorkflowRuns], RepoWorkflowRunsBookmark]: + """ + This method returns all workflow runs of a repo's workflow. After the bookmark date. + :param org_repo: OrgRepo object to get workflow runs for + :param repo_workflow: RepoWorkflow object to get workflow runs for + :param bookmark: Bookmark object to get all workflow runs after this date + :return: List of RepoWorkflowRuns objects, RepoWorkflowRunsBookmark object + """ + pass diff --git a/backend/analytics_server/dora/service/workflows/sync/etl_workflows_factory.py b/backend/analytics_server/dora/service/workflows/sync/etl_workflows_factory.py new file mode 100644 index 000000000..8aadb02d8 --- /dev/null +++ b/backend/analytics_server/dora/service/workflows/sync/etl_workflows_factory.py @@ -0,0 +1,15 @@ +from dora.service.workflows.sync.etl_github_actions_handler import ( + get_github_actions_etl_handler, +) +from dora.service.workflows.sync.etl_provider_handler import WorkflowProviderETLHandler +from dora.store.models.code import RepoWorkflowProviders + + +class WorkflowETLFactory: + def __init__(self, org_id: str): + self.org_id = org_id + + def __call__(self, provider: str) -> WorkflowProviderETLHandler: + if provider == RepoWorkflowProviders.GITHUB_ACTIONS.name: + return get_github_actions_etl_handler(self.org_id) + raise NotImplementedError(f"Unknown provider - {provider}") diff --git a/backend/analytics_server/dora/service/workflows/workflow_filter.py b/backend/analytics_server/dora/service/workflows/workflow_filter.py new file mode 100644 index 000000000..a693687ca --- /dev/null +++ b/backend/analytics_server/dora/service/workflows/workflow_filter.py @@ -0,0 +1,52 @@ +import json + +from typing import List, Dict + +from sqlalchemy import or_ + +from dora.store.models.code.workflows.filter import WorkflowFilter + + +class ParseWorkflowFilterProcessor: + def apply(self, workflow_filter: Dict = None) -> WorkflowFilter: + head_branches: List[str] = self._parse_head_branches(workflow_filter) + repo_filters: Dict[str, Dict] = self._parse_repo_filters(workflow_filter) + + return WorkflowFilter( + head_branches=head_branches, + repo_filters=repo_filters, + ) + + def _parse_head_branches(self, workflow_filter: Dict) -> List[str]: + return workflow_filter.get("head_branches") + + def _parse_repo_filters(self, workflow_filter: Dict) -> Dict[str, Dict]: + repo_filters: Dict[str, Dict] = workflow_filter.get("repo_filters") + if repo_filters: + for repo_id, repo_filter in repo_filters.items(): + repo_head_branches: List[str] = self._parse_repo_head_branches( + repo_filter + ) + repo_filters[repo_id]["head_branches"] = repo_head_branches + return repo_filters + + def _parse_repo_head_branches(self, repo_filter: Dict[str, any]) -> List[str]: + repo_head_branches: List[str] = repo_filter.get("head_branches") + if not repo_head_branches: + return [] + return repo_head_branches + + +class WorkflowFilterProcessor: + def __init__(self, parse_workflow_filter_processor: ParseWorkflowFilterProcessor): + self.parse_workflow_filter_processor = parse_workflow_filter_processor + + def create_workflow_filter_from_json_string( + self, filter_data: str + ) -> WorkflowFilter: + filter_data = filter_data or "{}" + return self.parse_workflow_filter_processor.apply(json.loads(filter_data)) + + +def get_workflow_filter_processor() -> WorkflowFilterProcessor: + return WorkflowFilterProcessor(ParseWorkflowFilterProcessor()) diff --git a/backend/analytics_server/dora/store/__init__.py b/backend/analytics_server/dora/store/__init__.py new file mode 100644 index 000000000..0ec5170c5 --- /dev/null +++ b/backend/analytics_server/dora/store/__init__.py @@ -0,0 +1,36 @@ +from os import getenv + +from flask_sqlalchemy import SQLAlchemy + +from dora.utils.log import LOG + +db = SQLAlchemy() + + +def configure_db_with_app(app): + + DB_HOST = getenv("DB_HOST") + DB_PORT = getenv("DB_PORT") + DB_USER = getenv("DB_USER") + DB_PASS = getenv("DB_PASS") + DB_NAME = getenv("DB_NAME") + ENVIRONMENT = getenv("ENVIRONMENT", "local") + + connection_uri = f"postgresql://{DB_USER}:{DB_PASS}@{DB_HOST}:{DB_PORT}/{DB_NAME}?application_name=dora--{ENVIRONMENT}" + + app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False + app.config["SQLALCHEMY_DATABASE_URI"] = connection_uri + app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {"pool_size": 10, "max_overflow": 5} + db.init_app(app) + + +def rollback_on_exc(func): + def wrapper(self, *args, **kwargs): + try: + return func(self, *args, **kwargs) + except Exception as e: + self._db.session.rollback() + LOG.error(f"Error in {func.__name__} - {str(e)}") + raise + + return wrapper diff --git a/backend/analytics_server/dora/store/initialise_db.py b/backend/analytics_server/dora/store/initialise_db.py new file mode 100644 index 000000000..7439662b8 --- /dev/null +++ b/backend/analytics_server/dora/store/initialise_db.py @@ -0,0 +1,27 @@ +from dora.store import db +from dora.store.models import Organization +from dora.utils.string import uuid4_str +from dora.utils.time import time_now + + +def initialize_database(app): + with app.app_context(): + default_org = ( + db.session.query(Organization) + .filter(Organization.name == "default") + .one_or_none() + ) + if default_org: + return + default_org = Organization( + id=uuid4_str(), + name="default", + domain="default", + created_at=time_now(), + ) + db.session.add(default_org) + db.session.commit() + + +if __name__ == "__main__": + initialize_database() diff --git a/backend/analytics_server/dora/store/models/__init__.py b/backend/analytics_server/dora/store/models/__init__.py new file mode 100644 index 000000000..dca8100bd --- /dev/null +++ b/backend/analytics_server/dora/store/models/__init__.py @@ -0,0 +1,7 @@ +from .core import Organization, Team, Users +from .integrations import Integration, UserIdentity, UserIdentityProvider +from .settings import ( + EntityType, + Settings, + SettingType, +) diff --git a/backend/analytics_server/dora/store/models/code/__init__.py b/backend/analytics_server/dora/store/models/code/__init__.py new file mode 100644 index 000000000..3552af427 --- /dev/null +++ b/backend/analytics_server/dora/store/models/code/__init__.py @@ -0,0 +1,31 @@ +from .enums import ( + CodeProvider, + BookmarkType, + PullRequestState, + PullRequestEventState, + PullRequestEventType, + PullRequestRevertPRMappingActorType, +) +from .filter import PRFilter +from .pull_requests import ( + PullRequest, + PullRequestEvent, + PullRequestCommit, + PullRequestRevertPRMapping, +) +from .repository import ( + OrgRepo, + TeamRepos, + RepoSyncLogs, + Bookmark, + BookmarkMergeToDeployBroker, +) +from .workflows import ( + RepoWorkflow, + RepoWorkflowRuns, + RepoWorkflowRunsBookmark, + RepoWorkflowType, + RepoWorkflowProviders, + RepoWorkflowRunsStatus, + WorkflowFilter, +) diff --git a/backend/analytics_server/dora/store/models/code/enums.py b/backend/analytics_server/dora/store/models/code/enums.py new file mode 100644 index 000000000..5c9512aac --- /dev/null +++ b/backend/analytics_server/dora/store/models/code/enums.py @@ -0,0 +1,35 @@ +from enum import Enum + + +class CodeProvider(Enum): + GITHUB = "github" + + +class BookmarkType(Enum): + PR = "PR" + + +class TeamReposDeploymentType(Enum): + WORKFLOW = "WORKFLOW" + PR_MERGE = "PR_MERGE" + + +class PullRequestState(Enum): + OPEN = "OPEN" + CLOSED = "CLOSED" + MERGED = "MERGED" + + +class PullRequestEventState(Enum): + CHANGES_REQUESTED = "CHANGES_REQUESTED" + APPROVED = "APPROVED" + COMMENTED = "COMMENTED" + + +class PullRequestEventType(Enum): + REVIEW = "REVIEW" + + +class PullRequestRevertPRMappingActorType(Enum): + SYSTEM = "SYSTEM" + USER = "USER" diff --git a/backend/analytics_server/dora/store/models/code/filter.py b/backend/analytics_server/dora/store/models/code/filter.py new file mode 100644 index 000000000..422f8fd3b --- /dev/null +++ b/backend/analytics_server/dora/store/models/code/filter.py @@ -0,0 +1,98 @@ +from dataclasses import dataclass +from typing import List, Dict + +from sqlalchemy import and_, or_ + +from dora.store.models.code.pull_requests import PullRequest + + +@dataclass +class PRFilter: + authors: List[str] = None + base_branches: List[str] = None + repo_filters: Dict[str, Dict] = None + excluded_pr_ids: List[str] = None + max_cycle_time: int = None + + class RepoFilter: + def __init__(self, repo_id: str, repo_filters=None): + if repo_filters is None: + repo_filters = {} + self.repo_id = repo_id + self.base_branches = repo_filters.get("base_branches", []) + + @property + def filter_query(self): + def _repo_id_query(): + if not self.repo_id: + raise ValueError("repo_id is required") + return PullRequest.repo_id == self.repo_id + + def _base_branch_query(): + if not self.base_branches: + return None + return or_( + PullRequest.base_branch.op("~")(term) + for term in self.base_branches + if term is not None + ) + + conditions = { + "repo_id": _repo_id_query(), + "base_branches": _base_branch_query(), + } + queries = [ + conditions[x] + for x in self.__dict__.keys() + if getattr(self, x) is not None and conditions[x] is not None + ] + if not queries: + return None + return and_(*queries) + + @property + def filter_query(self) -> List: + def _base_branch_query(): + if not self.base_branches: + return None + + return or_( + PullRequest.base_branch.op("~")(term) for term in self.base_branches + ) + + def _repo_filters_query(): + if not self.repo_filters: + return None + + return or_( + self.RepoFilter(repo_id, repo_filters).filter_query + for repo_id, repo_filters in self.repo_filters.items() + if repo_filters + ) + + def _excluded_pr_ids_query(): + if not self.excluded_pr_ids: + return None + + return PullRequest.id.notin_(self.excluded_pr_ids) + + def _include_prs_below_max_cycle_time(): + if not self.max_cycle_time: + return None + + return and_( + PullRequest.cycle_time != None, + PullRequest.cycle_time < self.max_cycle_time, + ) + + conditions = { + "base_branches": _base_branch_query(), + "repo_filters": _repo_filters_query(), + "excluded_pr_ids": _excluded_pr_ids_query(), + "max_cycle_time": _include_prs_below_max_cycle_time(), + } + return [ + conditions[x] + for x in self.__dict__.keys() + if getattr(self, x) is not None and conditions[x] is not None + ] diff --git a/backend/analytics_server/dora/store/models/code/pull_requests.py b/backend/analytics_server/dora/store/models/code/pull_requests.py new file mode 100644 index 000000000..0be275602 --- /dev/null +++ b/backend/analytics_server/dora/store/models/code/pull_requests.py @@ -0,0 +1,155 @@ +from datetime import datetime + +from sqlalchemy import func +from sqlalchemy.dialects.postgresql import UUID, JSONB, ARRAY, ENUM + +from dora.store import db +from dora.store.models.code.enums import ( + PullRequestEventType, + PullRequestState, + PullRequestRevertPRMappingActorType, +) + + +class PullRequest(db.Model): + __tablename__ = "PullRequest" + + id = db.Column(UUID(as_uuid=True), primary_key=True) + repo_id = db.Column(UUID(as_uuid=True), db.ForeignKey("OrgRepo.id")) + title = db.Column(db.String) + url = db.Column(db.String) + number = db.Column(db.String) + author = db.Column(db.String) + state = db.Column(ENUM(PullRequestState)) + requested_reviews = db.Column(ARRAY(db.String)) + base_branch = db.Column(db.String) + head_branch = db.Column(db.String) + data = db.Column(JSONB) + created_at = db.Column(db.DateTime(timezone=True)) + updated_at = db.Column(db.DateTime(timezone=True)) + state_changed_at = db.Column(db.DateTime(timezone=True)) + first_response_time = db.Column(db.Integer) + rework_time = db.Column(db.Integer) + merge_time = db.Column(db.Integer) + cycle_time = db.Column(db.Integer) + reviewers = db.Column(ARRAY(db.String)) + meta = db.Column(JSONB) + provider = db.Column(db.String) + rework_cycles = db.Column(db.Integer, default=0) + first_commit_to_open = db.Column(db.Integer) + merge_to_deploy = db.Column(db.Integer) + lead_time = db.Column(db.Integer) + merge_commit_sha = db.Column(db.String) + created_in_db_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_in_db_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + def __eq__(self, other): + return self.id == other.id + + def __lt__(self, other): + return self.id < other.id + + def __hash__(self): + return hash(self.id) + + @property + def commits(self) -> int: + return self.meta.get("code_stats", {}).get("commits", 0) + + @property + def additions(self) -> int: + return self.meta.get("code_stats", {}).get("additions", 0) + + @property + def deletions(self) -> int: + return self.meta.get("code_stats", {}).get("deletions", 0) + + @property + def changed_files(self) -> int: + return self.meta.get("code_stats", {}).get("changed_files", 0) + + @property + def comments(self) -> int: + return self.meta.get("code_stats", {}).get("comments", 0) + + @property + def username(self) -> str: + return self.meta.get("user_profile", {}).get("username", "") + + +class PullRequestEvent(db.Model): + __tablename__ = "PullRequestEvent" + + id = db.Column(UUID(as_uuid=True), primary_key=True) + pull_request_id = db.Column(UUID(as_uuid=True), db.ForeignKey("PullRequest.id")) + type = db.Column(ENUM(PullRequestEventType)) + data = db.Column(JSONB) + created_at = db.Column(db.DateTime(timezone=True)) + idempotency_key = db.Column(db.String) + org_repo_id = db.Column(UUID(as_uuid=True), db.ForeignKey("OrgRepo.id")) + actor_username = db.Column(db.String) + created_in_db_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_in_db_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + @property + def state(self): + if self.type in [ + PullRequestEventType.REVIEW.value, + PullRequestEventType.REVIEW, + ]: + return self.data.get("state", "") + + return "" + + +class PullRequestCommit(db.Model): + __tablename__ = "PullRequestCommit" + + hash = db.Column(db.String, primary_key=True) + pull_request_id = db.Column(UUID(as_uuid=True), db.ForeignKey("PullRequest.id")) + message = db.Column(db.String) + url = db.Column(db.String) + data = db.Column(JSONB) + author = db.Column(db.String) + created_at = db.Column(db.DateTime(timezone=True)) + org_repo_id = db.Column(UUID(as_uuid=True), db.ForeignKey("OrgRepo.id")) + created_in_db_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_in_db_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class PullRequestRevertPRMapping(db.Model): + __tablename__ = "PullRequestRevertPRMapping" + + pr_id = db.Column( + UUID(as_uuid=True), + db.ForeignKey("PullRequest.id"), + primary_key=True, + nullable=False, + ) + actor_type = db.Column( + ENUM(PullRequestRevertPRMappingActorType), primary_key=True, nullable=False + ) + actor = db.Column(UUID(as_uuid=True), db.ForeignKey("Users.id")) + reverted_pr = db.Column( + UUID(as_uuid=True), db.ForeignKey("PullRequest.id"), nullable=False + ) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + def __hash__(self): + return hash((self.pr_id, self.reverted_pr)) + + def __eq__(self, other): + return ( + isinstance(other, PullRequestRevertPRMapping) + and self.pr_id == other.pr_id + and self.reverted_pr == other.reverted_pr + ) diff --git a/backend/analytics_server/dora/store/models/code/repository.py b/backend/analytics_server/dora/store/models/code/repository.py new file mode 100644 index 000000000..12ec95e18 --- /dev/null +++ b/backend/analytics_server/dora/store/models/code/repository.py @@ -0,0 +1,108 @@ +import uuid +from datetime import datetime +from typing import Tuple + +import pytz +from sqlalchemy import func +from sqlalchemy.dialects.postgresql import UUID, JSONB, ARRAY, ENUM + +from dora.store import db +from dora.store.models.code.enums import ( + CodeProvider, + BookmarkType, + TeamReposDeploymentType, +) + + +class OrgRepo(db.Model): + __tablename__ = "OrgRepo" + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + org_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Organization.id")) + name = db.Column(db.String) + provider = db.Column(db.String) + org_name = db.Column(db.String) + default_branch = db.Column(db.String) + language = db.Column(db.String) + contributors = db.Column(JSONB) + idempotency_key = db.Column(db.String) + slug = db.Column(db.String) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + is_active = db.Column(db.Boolean, default=True) + + @property + def url(self): + if self.provider == CodeProvider.GITHUB.value: + return f"https://www.github.com/{self.org_name}/{self.name}" + + raise NotImplementedError(f"URL not implemented for {self.provider}") + + @property + def contributor_count(self) -> [Tuple[str, int]]: + if not self.contributors: + return [] + + return self.contributors.get("contributions", []) + + def __hash__(self): + return hash(self.id) + + +class TeamRepos(db.Model): + __tablename__ = "TeamRepos" + + team_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Team.id"), primary_key=True) + org_repo_id = db.Column( + UUID(as_uuid=True), db.ForeignKey("OrgRepo.id"), primary_key=True + ) + prod_branch = db.Column(db.String) + prod_branches = db.Column(ARRAY(db.String)) + deployment_type = db.Column( + ENUM(TeamReposDeploymentType), default=TeamReposDeploymentType.PR_MERGE + ) + is_active = db.Column(db.Boolean, default=True) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class RepoSyncLogs(db.Model): + __tablename__ = "RepoSyncLogs" + + repo_id = db.Column( + UUID(as_uuid=True), db.ForeignKey("OrgRepo.id"), primary_key=True + ) + synced_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + + +class Bookmark(db.Model): + __tablename__ = "Bookmark" + + repo_id = db.Column(UUID(as_uuid=True), primary_key=True) + type = db.Column(ENUM(BookmarkType), primary_key=True) + bookmark = db.Column(db.String) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class BookmarkMergeToDeployBroker(db.Model): + __tablename__ = "BookmarkMergeToDeployBroker" + + repo_id = db.Column(UUID(as_uuid=True), primary_key=True) + bookmark = db.Column(db.String) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + @property + def bookmark_date(self): + if not self.bookmark: + return None + return datetime.fromisoformat(self.bookmark).astimezone(tz=pytz.UTC) diff --git a/backend/analytics_server/dora/store/models/code/workflows/__init__.py b/backend/analytics_server/dora/store/models/code/workflows/__init__.py new file mode 100644 index 000000000..75f95af15 --- /dev/null +++ b/backend/analytics_server/dora/store/models/code/workflows/__init__.py @@ -0,0 +1,3 @@ +from .enums import RepoWorkflowType, RepoWorkflowProviders, RepoWorkflowRunsStatus +from .filter import WorkflowFilter +from .workflows import RepoWorkflow, RepoWorkflowRuns, RepoWorkflowRunsBookmark diff --git a/backend/analytics_server/dora/store/models/code/workflows/enums.py b/backend/analytics_server/dora/store/models/code/workflows/enums.py new file mode 100644 index 000000000..cfcbd7584 --- /dev/null +++ b/backend/analytics_server/dora/store/models/code/workflows/enums.py @@ -0,0 +1,32 @@ +from enum import Enum + + +class RepoWorkflowProviders(Enum): + GITHUB_ACTIONS = "github" + CIRCLE_CI = "circle_ci" + + @classmethod + def get_workflow_providers(cls): + return [v for v in cls.__members__.values()] + + @classmethod + def get_workflow_providers_values(cls): + return [v.value for v in cls.__members__.values()] + + @classmethod + def get_enum(cls, provider: str): + for v in cls.__members__.values(): + if provider == v.value: + return v + return None + + +class RepoWorkflowType(Enum): + DEPLOYMENT = "DEPLOYMENT" + + +class RepoWorkflowRunsStatus(Enum): + SUCCESS = "SUCCESS" + FAILURE = "FAILURE" + PENDING = "PENDING" + CANCELLED = "CANCELLED" diff --git a/backend/analytics_server/dora/store/models/code/workflows/filter.py b/backend/analytics_server/dora/store/models/code/workflows/filter.py new file mode 100644 index 000000000..baf220efa --- /dev/null +++ b/backend/analytics_server/dora/store/models/code/workflows/filter.py @@ -0,0 +1,81 @@ +from dataclasses import dataclass +from operator import and_ +from typing import List, Dict + +from sqlalchemy import or_ + +from .workflows import RepoWorkflowRuns, RepoWorkflow + + +class RepoWorkflowFilter: + def __init__(self, repo_id: str, repo_filters=None): + if repo_filters is None: + repo_filters = {} + self.repo_id = repo_id + self.head_branches = repo_filters.get("head_branches", []) + + @property + def filter_query(self): + def _repo_id_query(): + if not self.repo_id: + raise ValueError("repo_id is required") + return RepoWorkflow.org_repo_id == self.repo_id + + def _head_branches_query(): + if not self.head_branches: + return None + return or_( + RepoWorkflowRuns.head_branch.op("~")(term) + for term in self.head_branches + if term is not None + ) + + conditions = { + "repo_id": _repo_id_query(), + "head_branches": _head_branches_query(), + } + queries = [ + conditions[x] + for x in self.__dict__.keys() + if getattr(self, x) is not None and conditions[x] is not None + ] + if not queries: + return None + return and_(*queries) + + +@dataclass +class WorkflowFilter: + head_branches: List[str] = None + repo_filters: Dict[str, Dict] = None + + @property + def filter_query(self) -> List: + def _head_branches_query(): + if not self.head_branches: + return None + + return or_( + RepoWorkflowRuns.head_branch.op("~")(term) + for term in self.head_branches + ) + + def _repo_filters_query(): + if not self.repo_filters: + return None + + return or_( + RepoWorkflowFilter(repo_id, repo_filters).filter_query + for repo_id, repo_filters in self.repo_filters.items() + if repo_filters + ) + + conditions = { + "head_branches": _head_branches_query(), + "repo_filters": _repo_filters_query(), + } + return [ + conditions[x] + for x in self.__dict__.keys() + if getattr(self, x) is not None and conditions[x] is not None + ] diff --git a/backend/analytics_server/dora/store/models/code/workflows/workflows.py b/backend/analytics_server/dora/store/models/code/workflows/workflows.py new file mode 100644 index 000000000..e02be6b1f --- /dev/null +++ b/backend/analytics_server/dora/store/models/code/workflows/workflows.py @@ -0,0 +1,62 @@ +import uuid + +from sqlalchemy import func +from sqlalchemy.dialects.postgresql import UUID, JSONB, ENUM + +from dora.store import db +from dora.store.models.code.workflows.enums import ( + RepoWorkflowType, + RepoWorkflowProviders, + RepoWorkflowRunsStatus, +) + + +class RepoWorkflow(db.Model): + __tablename__ = "RepoWorkflow" + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + org_repo_id = db.Column(UUID(as_uuid=True), db.ForeignKey("OrgRepo.id")) + type = db.Column(ENUM(RepoWorkflowType)) + provider = db.Column(ENUM(RepoWorkflowProviders)) + provider_workflow_id = db.Column(db.String, nullable=False) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + meta = db.Column(JSONB, default="{}") + is_active = db.Column(db.Boolean, default=True) + name = db.Column(db.String) + + +class RepoWorkflowRuns(db.Model): + __tablename__ = "RepoWorkflowRuns" + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + repo_workflow_id = db.Column(UUID(as_uuid=True), db.ForeignKey("RepoWorkflow.id")) + provider_workflow_run_id = db.Column(db.String, nullable=False) + event_actor = db.Column(db.String) + head_branch = db.Column(db.String) + status = db.Column(ENUM(RepoWorkflowRunsStatus)) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + conducted_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + meta = db.Column(JSONB, default="{}") + duration = db.Column(db.Integer) + html_url = db.Column(db.String) + + def __hash__(self): + return hash(self.id) + + +class RepoWorkflowRunsBookmark(db.Model): + __tablename__ = "RepoWorkflowRunsBookmark" + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + repo_workflow_id = db.Column(UUID(as_uuid=True), db.ForeignKey("RepoWorkflow.id")) + bookmark = db.Column(db.String) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) diff --git a/backend/analytics_server/dora/store/models/core/__init__.py b/backend/analytics_server/dora/store/models/core/__init__.py new file mode 100644 index 000000000..6c21e1bb6 --- /dev/null +++ b/backend/analytics_server/dora/store/models/core/__init__.py @@ -0,0 +1,3 @@ +from .organization import Organization +from .teams import Team +from .users import Users diff --git a/backend/analytics_server/dora/store/models/core/organization.py b/backend/analytics_server/dora/store/models/core/organization.py new file mode 100644 index 000000000..c275de442 --- /dev/null +++ b/backend/analytics_server/dora/store/models/core/organization.py @@ -0,0 +1,23 @@ +from sqlalchemy.dialects.postgresql import UUID, ARRAY + +from dora.store import db + + +class Organization(db.Model): + __tablename__ = "Organization" + + id = db.Column(UUID(as_uuid=True), primary_key=True) + name = db.Column(db.String) + created_at = db.Column(db.DateTime(timezone=True)) + domain = db.Column(db.String) + other_domains = db.Column(ARRAY(db.String)) + + def __eq__(self, other): + + if isinstance(other, Organization): + return self.id == other.id + + return False + + def __hash__(self): + return hash(self.id) diff --git a/backend/analytics_server/dora/store/models/core/teams.py b/backend/analytics_server/dora/store/models/core/teams.py new file mode 100644 index 000000000..13884113b --- /dev/null +++ b/backend/analytics_server/dora/store/models/core/teams.py @@ -0,0 +1,26 @@ +import uuid + +from sqlalchemy import ( + func, +) +from sqlalchemy.dialects.postgresql import UUID, ARRAY + +from dora.store import db + + +class Team(db.Model): + __tablename__ = "Team" + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + org_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Organization.id")) + name = db.Column(db.String) + member_ids = db.Column(ARRAY(UUID(as_uuid=True)), nullable=False) + manager_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Users.id")) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + is_deleted = db.Column(db.Boolean, default=False) + + def __hash__(self): + return hash(self.id) diff --git a/backend/analytics_server/dora/store/models/core/users.py b/backend/analytics_server/dora/store/models/core/users.py new file mode 100644 index 000000000..e34e005b8 --- /dev/null +++ b/backend/analytics_server/dora/store/models/core/users.py @@ -0,0 +1,21 @@ +from sqlalchemy import ( + func, +) +from sqlalchemy.dialects.postgresql import UUID + +from dora.store import db + + +class Users(db.Model): + __tablename__ = "Users" + + id = db.Column(UUID(as_uuid=True), primary_key=True) + org_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Organization.id")) + name = db.Column(db.String) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + primary_email = db.Column(db.String) + is_deleted = db.Column(db.Boolean, default=False) + avatar_url = db.Column(db.String) diff --git a/backend/analytics_server/dora/store/models/incidents/__init__.py b/backend/analytics_server/dora/store/models/incidents/__init__.py new file mode 100644 index 000000000..aef271901 --- /dev/null +++ b/backend/analytics_server/dora/store/models/incidents/__init__.py @@ -0,0 +1,15 @@ +from .enums import ( + IncidentType, + IncidentBookmarkType, + IncidentProvider, + ServiceStatus, + IncidentStatus, + IncidentSource, +) +from .filter import IncidentFilter +from .incidents import ( + Incident, + IncidentOrgIncidentServiceMap, + IncidentsBookmark, +) +from .services import OrgIncidentService, TeamIncidentService diff --git a/backend/analytics_server/dora/store/models/incidents/enums.py b/backend/analytics_server/dora/store/models/incidents/enums.py new file mode 100644 index 000000000..932945f8a --- /dev/null +++ b/backend/analytics_server/dora/store/models/incidents/enums.py @@ -0,0 +1,35 @@ +from enum import Enum + + +class IncidentProvider(Enum): + GITHUB = "github" + + +class IncidentSource(Enum): + INCIDENT_SERVICE = "INCIDENT_SERVICE" + INCIDENT_TEAM = "INCIDENT_TEAM" + GIT_REPO = "GIT_REPO" + + +class ServiceStatus(Enum): + DISABLED = "disabled" + ACTIVE = "active" + WARNING = "warning" + CRITICAL = "critical" + MAINTENANCE = "maintenance" + + +class IncidentStatus(Enum): + TRIGGERED = "triggered" + ACKNOWLEDGED = "acknowledged" + RESOLVED = "resolved" + + +class IncidentType(Enum): + INCIDENT = "INCIDENT" + REVERT_PR = "REVERT_PR" + ALERT = "ALERT" + + +class IncidentBookmarkType(Enum): + SERVICE = "SERVICE" diff --git a/backend/analytics_server/dora/store/models/incidents/filter.py b/backend/analytics_server/dora/store/models/incidents/filter.py new file mode 100644 index 000000000..d041c1f1a --- /dev/null +++ b/backend/analytics_server/dora/store/models/incidents/filter.py @@ -0,0 +1,45 @@ +from dataclasses import dataclass +from typing import List + +from sqlalchemy import or_ + +from dora.store.models.incidents.incidents import Incident + + +@dataclass +class IncidentFilter: + """Dataclass for filtering incidents.""" + + title_filter_substrings: List[str] = None + incident_types: List[str] = None + + @property + def filter_query(self) -> List: + def _title_filter_substrings_query(): + if not self.title_filter_substrings: + return None + + return or_( + Incident.title.contains(substring, autoescape=True) + for substring in self.title_filter_substrings + ) + + def _incident_type_query(): + if not self.incident_types: + return None + + return or_( + Incident.incident_type == incident_type + for incident_type in self.incident_types + ) + + conditions = { + "title_filter_substrings": _title_filter_substrings_query(), + "incident_types": _incident_type_query(), + } + + return [ + conditions[x] + for x in self.__dict__.keys() + if getattr(self, x) is not None and conditions[x] is not None + ] diff --git a/backend/analytics_server/dora/store/models/incidents/incidents.py b/backend/analytics_server/dora/store/models/incidents/incidents.py new file mode 100644 index 000000000..464744e0c --- /dev/null +++ b/backend/analytics_server/dora/store/models/incidents/incidents.py @@ -0,0 +1,59 @@ +from sqlalchemy import ( + func, +) +from sqlalchemy.dialects.postgresql import UUID, ARRAY, JSONB, ENUM + +from dora.store import db +from dora.store.models.incidents.enums import IncidentType, IncidentBookmarkType + + +class Incident(db.Model): + __tablename__ = "Incident" + + id = db.Column(UUID(as_uuid=True), primary_key=True) + provider = db.Column(db.String) + key = db.Column(db.String) + incident_number = db.Column(db.Integer) + title = db.Column(db.String) + status = db.Column(db.String) + creation_date = db.Column(db.DateTime(timezone=True)) + acknowledged_date = db.Column(db.DateTime(timezone=True)) + resolved_date = db.Column(db.DateTime(timezone=True)) + assigned_to = db.Column(db.String) + assignees = db.Column(ARRAY(db.String)) + incident_type = db.Column(ENUM(IncidentType), default=IncidentType.INCIDENT) + meta = db.Column(JSONB, default={}) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + def __hash__(self): + return hash(self.id) + + +class IncidentOrgIncidentServiceMap(db.Model): + __tablename__ = "IncidentOrgIncidentServiceMap" + + incident_id = db.Column( + UUID(as_uuid=True), db.ForeignKey("Incident.id"), primary_key=True + ) + service_id = db.Column( + UUID(as_uuid=True), db.ForeignKey("OrgIncidentService.id"), primary_key=True + ) + + +class IncidentsBookmark(db.Model): + __tablename__ = "IncidentsBookmark" + + id = db.Column(UUID(as_uuid=True), primary_key=True) + provider = db.Column(db.String) + entity_id = db.Column(UUID(as_uuid=True)) + entity_type = db.Column( + ENUM(IncidentBookmarkType), default=IncidentBookmarkType.SERVICE + ) + bookmark = db.Column(db.DateTime(timezone=True), server_default=func.now()) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) diff --git a/backend/analytics_server/dora/store/models/incidents/services.py b/backend/analytics_server/dora/store/models/incidents/services.py new file mode 100644 index 000000000..14b487c32 --- /dev/null +++ b/backend/analytics_server/dora/store/models/incidents/services.py @@ -0,0 +1,45 @@ +from sqlalchemy import ( + func, +) +from sqlalchemy.dialects.postgresql import UUID, ARRAY, JSONB, ENUM +from sqlalchemy.orm import relationship + +from dora.store import db +from dora.store.models.incidents import IncidentSource + + +class OrgIncidentService(db.Model): + __tablename__ = "OrgIncidentService" + + id = db.Column(UUID(as_uuid=True), primary_key=True) + org_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Organization.id")) + name = db.Column(db.String) + provider = db.Column(db.String) + key = db.Column(db.String) + auto_resolve_timeout = db.Column(db.Integer) + acknowledgement_timeout = db.Column(db.Integer) + created_by = db.Column(db.String) + provider_team_keys = db.Column(ARRAY(db.String)) + status = db.Column(db.String) + is_deleted = db.Column(db.Boolean, default=False) + meta = db.Column(JSONB, default={}) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + source_type = db.Column( + ENUM(IncidentSource), default=IncidentSource.INCIDENT_SERVICE, nullable=False + ) + + +class TeamIncidentService(db.Model): + __tablename__ = "TeamIncidentService" + + id = db.Column(UUID(as_uuid=True), primary_key=True) + team_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Team.id")) + service_id = db.Column(UUID(as_uuid=True), db.ForeignKey("OrgIncidentService.id")) + OrgIncidentService = relationship("OrgIncidentService", lazy="joined") + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) diff --git a/backend/analytics_server/dora/store/models/integrations/__init__.py b/backend/analytics_server/dora/store/models/integrations/__init__.py new file mode 100644 index 000000000..0192a8591 --- /dev/null +++ b/backend/analytics_server/dora/store/models/integrations/__init__.py @@ -0,0 +1,2 @@ +from .enums import UserIdentityProvider +from .integrations import Integration, UserIdentity diff --git a/backend/analytics_server/dora/store/models/integrations/enums.py b/backend/analytics_server/dora/store/models/integrations/enums.py new file mode 100644 index 000000000..6f4e8b449 --- /dev/null +++ b/backend/analytics_server/dora/store/models/integrations/enums.py @@ -0,0 +1,12 @@ +from enum import Enum + + +class UserIdentityProvider(Enum): + GITHUB = "github" + + @classmethod + def get_enum(self, provider: str): + for v in self.__members__.values(): + if provider == v.value: + return v + return None diff --git a/backend/analytics_server/dora/store/models/integrations/integrations.py b/backend/analytics_server/dora/store/models/integrations/integrations.py new file mode 100644 index 000000000..c0010e3f4 --- /dev/null +++ b/backend/analytics_server/dora/store/models/integrations/integrations.py @@ -0,0 +1,49 @@ +from sqlalchemy import ( + func, +) +from sqlalchemy.dialects.postgresql import UUID, ARRAY, JSONB + +from dora.store import db +from dora.store.models.integrations import UserIdentityProvider + + +class Integration(db.Model): + __tablename__ = "Integration" + + org_id = db.Column( + UUID(as_uuid=True), db.ForeignKey("Organization.id"), primary_key=True + ) + name = db.Column(db.String, primary_key=True) + generated_by = db.Column( + UUID(as_uuid=True), db.ForeignKey("Users.id"), nullable=True + ) + access_token_enc_chunks = db.Column(ARRAY(db.String)) + refresh_token_enc_chunks = db.Column(ARRAY(db.String)) + provider_meta = db.Column(JSONB) + scopes = db.Column(ARRAY(db.String)) + access_token_valid_till = db.Column(db.DateTime(timezone=True)) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class UserIdentity(db.Model): + __tablename__ = "UserIdentity" + + user_id = db.Column(UUID(as_uuid=True), primary_key=True) + provider = db.Column(db.String, primary_key=True) + token = db.Column(db.String) + username = db.Column(db.String) + refresh_token = db.Column(db.String) + org_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Organization.id")) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + meta = db.Column(JSONB) + + @property + def avatar_url(self): + if self.provider == UserIdentityProvider.GITHUB.value: + return f"https://github.com/{self.username}.png" diff --git a/backend/analytics_server/dora/store/models/settings/__init__.py b/backend/analytics_server/dora/store/models/settings/__init__.py new file mode 100644 index 000000000..450b0adce --- /dev/null +++ b/backend/analytics_server/dora/store/models/settings/__init__.py @@ -0,0 +1,2 @@ +from .configuration_settings import SettingType, Settings +from .enums import EntityType diff --git a/backend/analytics_server/dora/store/models/settings/configuration_settings.py b/backend/analytics_server/dora/store/models/settings/configuration_settings.py new file mode 100644 index 000000000..e5613ee11 --- /dev/null +++ b/backend/analytics_server/dora/store/models/settings/configuration_settings.py @@ -0,0 +1,33 @@ +from enum import Enum + +from sqlalchemy import func +from sqlalchemy.dialects.postgresql import UUID, ENUM, JSONB + +from dora.store import db +from dora.store.models.settings.enums import EntityType + +""" +All Data config settings will be stored in the below table. +""" + + +class SettingType(Enum): + INCIDENT_SETTING = "INCIDENT_SETTING" + INCIDENT_TYPES_SETTING = "INCIDENT_TYPES_SETTING" + INCIDENT_SOURCES_SETTING = "INCIDENT_SOURCES_SETTING" + EXCLUDED_PRS_SETTING = "EXCLUDED_PRS_SETTING" + + +class Settings(db.Model): + __tablename__ = "Settings" + + entity_id = db.Column(UUID(as_uuid=True), primary_key=True, nullable=False) + entity_type = db.Column(ENUM(EntityType), primary_key=True, nullable=False) + setting_type = db.Column(ENUM(SettingType), primary_key=True, nullable=False) + updated_by = db.Column(UUID(as_uuid=True), db.ForeignKey("Users.id")) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + data = db.Column(JSONB, default="{}") + is_deleted = db.Column(db.Boolean, default=False) diff --git a/backend/analytics_server/dora/store/models/settings/enums.py b/backend/analytics_server/dora/store/models/settings/enums.py new file mode 100644 index 000000000..caa351ebe --- /dev/null +++ b/backend/analytics_server/dora/store/models/settings/enums.py @@ -0,0 +1,7 @@ +from enum import Enum + + +class EntityType(Enum): + USER = "USER" + TEAM = "TEAM" + ORG = "ORG" diff --git a/backend/analytics_server/dora/store/repos/__init__.py b/backend/analytics_server/dora/store/repos/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/dora/store/repos/code.py b/backend/analytics_server/dora/store/repos/code.py new file mode 100644 index 000000000..512d303d5 --- /dev/null +++ b/backend/analytics_server/dora/store/repos/code.py @@ -0,0 +1,333 @@ +from datetime import datetime +from operator import and_ +from typing import Optional, List + +from sqlalchemy import or_ +from sqlalchemy.orm import defer + +from dora.store import db, rollback_on_exc +from dora.store.models.code import ( + PullRequest, + PullRequestEvent, + OrgRepo, + PullRequestRevertPRMapping, + PullRequestCommit, + Bookmark, + TeamRepos, + PullRequestState, + PRFilter, + BookmarkMergeToDeployBroker, +) +from dora.utils.time import Interval + + +class CodeRepoService: + def __init__(self): + self._db = db + + @rollback_on_exc + def get_active_org_repos(self, org_id: str) -> List[OrgRepo]: + return ( + self._db.session.query(OrgRepo) + .filter(OrgRepo.org_id == org_id, OrgRepo.is_active.is_(True)) + .all() + ) + + @rollback_on_exc + def update_org_repos(self, org_repos: List[OrgRepo]): + [self._db.session.merge(org_repo) for org_repo in org_repos] + self._db.session.commit() + + @rollback_on_exc + def save_pull_requests_data( + self, + pull_requests: List[PullRequest], + pull_request_commits: List[PullRequestCommit], + pull_request_events: List[PullRequestEvent], + ): + [self._db.session.merge(pull_request) for pull_request in pull_requests] + [ + self._db.session.merge(pull_request_commit) + for pull_request_commit in pull_request_commits + ] + [ + self._db.session.merge(pull_request_event) + for pull_request_event in pull_request_events + ] + self._db.session.commit() + + @rollback_on_exc + def update_prs(self, prs: List[PullRequest]): + [self._db.session.merge(pr) for pr in prs] + self._db.session.commit() + + @rollback_on_exc + def save_revert_pr_mappings( + self, revert_pr_mappings: List[PullRequestRevertPRMapping] + ): + [self._db.session.merge(revert_pr_map) for revert_pr_map in revert_pr_mappings] + self._db.session.commit() + + @rollback_on_exc + def get_org_repo_bookmark(self, org_repo: OrgRepo, bookmark_type): + return ( + self._db.session.query(Bookmark) + .filter( + and_( + Bookmark.repo_id == org_repo.id, + Bookmark.type == bookmark_type.value, + ) + ) + .one_or_none() + ) + + @rollback_on_exc + def update_org_repo_bookmark(self, bookmark: Bookmark): + self._db.session.merge(bookmark) + self._db.session.commit() + + @rollback_on_exc + def get_repo_by_id(self, repo_id: str) -> Optional[OrgRepo]: + return ( + self._db.session.query(OrgRepo).filter(OrgRepo.id == repo_id).one_or_none() + ) + + @rollback_on_exc + def get_repo_pr_by_number(self, repo_id: str, pr_number) -> Optional[PullRequest]: + return ( + self._db.session.query(PullRequest) + .options(defer(PullRequest.data)) + .filter( + and_( + PullRequest.repo_id == repo_id, PullRequest.number == str(pr_number) + ) + ) + .one_or_none() + ) + + @rollback_on_exc + def get_pr_events(self, pr_model: PullRequest): + if not pr_model: + return [] + + pr_events = ( + self._db.session.query(PullRequestEvent) + .options(defer(PullRequestEvent.data)) + .filter(PullRequestEvent.pull_request_id == pr_model.id) + .all() + ) + return pr_events + + @rollback_on_exc + def get_prs_by_ids(self, pr_ids: List[str]): + query = ( + self._db.session.query(PullRequest) + .options(defer(PullRequest.data)) + .filter(PullRequest.id.in_(pr_ids)) + ) + return query.all() + + @rollback_on_exc + def get_prs_by_head_branch_match_strings( + self, repo_ids: List[str], match_strings: List[str] + ) -> List[PullRequest]: + query = ( + self._db.session.query(PullRequest) + .options(defer(PullRequest.data)) + .filter( + and_( + PullRequest.repo_id.in_(repo_ids), + or_( + *[ + PullRequest.head_branch.ilike(f"{match_string}%") + for match_string in match_strings + ] + ), + ) + ) + .order_by(PullRequest.updated_in_db_at.desc()) + ) + + return query.all() + + @rollback_on_exc + def get_reverted_prs_by_numbers( + self, repo_ids: List[str], numbers: List[str] + ) -> List[PullRequest]: + query = ( + self._db.session.query(PullRequest) + .options(defer(PullRequest.data)) + .filter( + and_( + PullRequest.repo_id.in_(repo_ids), + PullRequest.number.in_(numbers), + ) + ) + .order_by(PullRequest.updated_in_db_at.desc()) + ) + + return query.all() + + @rollback_on_exc + def get_active_team_repos_by_team_id(self, team_id: str) -> List[TeamRepos]: + return ( + self._db.session.query(TeamRepos) + .filter(TeamRepos.team_id == team_id, TeamRepos.is_active.is_(True)) + .all() + ) + + @rollback_on_exc + def get_active_team_repos_by_team_ids(self, team_ids: List[str]) -> List[TeamRepos]: + return ( + self._db.session.query(TeamRepos) + .filter(TeamRepos.team_id.in_(team_ids), TeamRepos.is_active.is_(True)) + .all() + ) + + @rollback_on_exc + def get_active_org_repos_by_ids(self, repo_ids: List[str]) -> List[OrgRepo]: + return ( + self._db.session.query(OrgRepo) + .filter(OrgRepo.id.in_(repo_ids), OrgRepo.is_active.is_(True)) + .all() + ) + + @rollback_on_exc + def get_prs_merged_in_interval( + self, + repo_ids: List[str], + interval: Interval, + pr_filter: PRFilter = None, + base_branches: List[str] = None, + has_non_null_mtd=False, + ) -> List[PullRequest]: + query = self._db.session.query(PullRequest).options(defer(PullRequest.data)) + + query = self._filter_prs_by_repo_ids(query, repo_ids) + query = self._filter_prs_merged_in_interval(query, interval) + + query = self._filter_prs(query, pr_filter) + query = self._filter_base_branch_on_regex(query, base_branches) + + if has_non_null_mtd: + query = query.filter(PullRequest.merge_to_deploy.is_not(None)) + + query = query.order_by(PullRequest.state_changed_at.asc()) + + return query.all() + + @rollback_on_exc + def get_pull_request_by_id(self, pr_id: str) -> PullRequest: + return ( + self._db.session.query(PullRequest) + .options(defer(PullRequest.data)) + .filter(PullRequest.id == pr_id) + .one_or_none() + ) + + @rollback_on_exc + def get_previous_pull_request(self, pull_request: PullRequest) -> PullRequest: + return ( + self._db.session.query(PullRequest) + .options(defer(PullRequest.data)) + .filter( + PullRequest.repo_id == pull_request.repo_id, + PullRequest.state_changed_at < pull_request.state_changed_at, + PullRequest.base_branch == pull_request.base_branch, + PullRequest.state == PullRequestState.MERGED, + ) + .order_by(PullRequest.state_changed_at.desc()) + .first() + ) + + @rollback_on_exc + def get_repos_by_ids(self, ids: List[str]) -> List[OrgRepo]: + if not ids: + return [] + + return self._db.session.query(OrgRepo).filter(OrgRepo.id.in_(ids)).all() + + @rollback_on_exc + def get_team_repos(self, team_id) -> List[OrgRepo]: + team_repos = ( + self._db.session.query(TeamRepos) + .filter(and_(TeamRepos.team_id == team_id, TeamRepos.is_active == True)) + .all() + ) + if not team_repos: + return [] + + team_repo_ids = [tr.org_repo_id for tr in team_repos] + return self.get_repos_by_ids(team_repo_ids) + + @rollback_on_exc + def get_merge_to_deploy_broker_bookmark( + self, repo_id: str + ) -> BookmarkMergeToDeployBroker: + return ( + self._db.session.query(BookmarkMergeToDeployBroker) + .filter(BookmarkMergeToDeployBroker.repo_id == repo_id) + .one_or_none() + ) + + @rollback_on_exc + def update_merge_to_deploy_broker_bookmark( + self, bookmark: BookmarkMergeToDeployBroker + ): + self._db.session.merge(bookmark) + self._db.session.commit() + + @rollback_on_exc + def get_prs_in_repo_merged_before_given_date_with_merge_to_deploy_as_null( + self, repo_id: str, to_time: datetime + ): + return ( + self._db.session.query(PullRequest) + .options(defer(PullRequest.data)) + .filter( + PullRequest.repo_id == repo_id, + PullRequest.state == PullRequestState.MERGED, + PullRequest.state_changed_at <= to_time, + PullRequest.merge_to_deploy.is_(None), + ) + .all() + ) + + @rollback_on_exc + def get_repo_revert_prs_mappings_updated_in_interval( + self, repo_id, from_time, to_time + ) -> List[PullRequestRevertPRMapping]: + query = ( + self._db.session.query(PullRequestRevertPRMapping) + .join(PullRequest, PullRequest.id == PullRequestRevertPRMapping.pr_id) + .filter( + PullRequest.repo_id == repo_id, + PullRequest.state == PullRequestState.MERGED, + PullRequestRevertPRMapping.updated_at.between(from_time, to_time), + ) + ) + query = query.order_by(PullRequest.updated_at.desc()) + + return query.all() + + def _filter_prs_by_repo_ids(self, query, repo_ids: List[str]): + return query.filter(PullRequest.repo_id.in_(repo_ids)) + + def _filter_prs_merged_in_interval(self, query, interval: Interval): + return query.filter( + PullRequest.state_changed_at.between(interval.from_time, interval.to_time), + PullRequest.state == PullRequestState.MERGED, + ) + + def _filter_prs(self, query, pr_filter: PRFilter): + if pr_filter: + query = query.filter(*pr_filter.filter_query) + return query + + def _filter_base_branch_on_regex(self, query, base_branches: List[str] = None): + if base_branches: + conditions = [ + PullRequest.base_branch.op("~")(term) for term in base_branches + ] + return query.filter(or_(*conditions)) + return query diff --git a/backend/analytics_server/dora/store/repos/core.py b/backend/analytics_server/dora/store/repos/core.py new file mode 100644 index 000000000..14d3d0fe5 --- /dev/null +++ b/backend/analytics_server/dora/store/repos/core.py @@ -0,0 +1,108 @@ +from typing import Optional, List + +from sqlalchemy import and_ + +from dora.store import db, rollback_on_exc +from dora.store.models import UserIdentityProvider, Integration +from dora.store.models.core import Organization, Team, Users +from dora.utils.cryptography import get_crypto_service + + +class CoreRepoService: + def __init__(self): + self._crypto = get_crypto_service() + self._db = db + + @rollback_on_exc + def get_org(self, org_id): + return ( + self._db.session.query(Organization) + .filter(Organization.id == org_id) + .one_or_none() + ) + + @rollback_on_exc + def get_org_by_name(self, org_name: str): + return ( + self._db.session.query(Organization) + .filter(Organization.name == org_name) + .one_or_none() + ) + + @rollback_on_exc + def get_team(self, team_id: str) -> Team: + return ( + self._db.session.query(Team) + .filter(Team.id == team_id, Team.is_deleted.is_(False)) + .one_or_none() + ) + + @rollback_on_exc + def delete_team(self, team_id: str): + + team = self._db.session.query(Team).filter(Team.id == team_id).one_or_none() + + if not team: + return None + + team.is_deleted = True + + self._db.session.merge(team) + self._db.session.commit() + return self._db.session.query(Team).filter(Team.id == team_id).one_or_none() + + @rollback_on_exc + def create_team(self, org_id: str, name: str, member_ids: List[str]) -> Team: + team = Team( + name=name, + org_id=org_id, + member_ids=member_ids or [], + is_deleted=False, + ) + self._db.session.add(team) + self._db.session.commit() + + return self.get_team(team.id) + + @rollback_on_exc + def update_team(self, team: Team) -> Team: + self._db.session.merge(team) + self._db.session.commit() + + return self.get_team(team.id) + + @rollback_on_exc + def get_user(self, user_id) -> Optional[Users]: + return self._db.session.query(Users).filter(Users.id == user_id).one_or_none() + + @rollback_on_exc + def get_users(self, user_ids: List[str]) -> List[Users]: + return ( + self._db.session.query(Users) + .filter(and_(Users.id.in_(user_ids), Users.is_deleted == False)) + .all() + ) + + @rollback_on_exc + def get_org_integrations_for_names(self, org_id: str, provider_names: List[str]): + return ( + self._db.session.query(Integration) + .filter( + and_(Integration.org_id == org_id, Integration.name.in_(provider_names)) + ) + .all() + ) + + @rollback_on_exc + def get_access_token(self, org_id, provider: UserIdentityProvider) -> Optional[str]: + user_identity: Integration = ( + self._db.session.query(Integration) + .filter( + and_(Integration.org_id == org_id, Integration.name == provider.value) + ) + .one_or_none() + ) + + if not user_identity or not user_identity.access_token_enc_chunks: + return None + return self._crypto.decrypt_chunks(user_identity.access_token_enc_chunks) diff --git a/backend/analytics_server/dora/store/repos/incidents.py b/backend/analytics_server/dora/store/repos/incidents.py new file mode 100644 index 000000000..532fa8c99 --- /dev/null +++ b/backend/analytics_server/dora/store/repos/incidents.py @@ -0,0 +1,148 @@ +from typing import List + +from sqlalchemy import and_ + +from dora.store import db, rollback_on_exc +from dora.store.models.incidents import ( + Incident, + IncidentFilter, + IncidentOrgIncidentServiceMap, + TeamIncidentService, + IncidentStatus, + IncidentType, + IncidentProvider, + OrgIncidentService, + IncidentsBookmark, + IncidentBookmarkType, +) +from dora.utils.time import Interval + + +class IncidentsRepoService: + def __init__(self): + self._db = db + + @rollback_on_exc + def get_org_incident_services(self, org_id: str) -> List[OrgIncidentService]: + return ( + self._db.session.query(OrgIncidentService) + .filter(OrgIncidentService.org_id == org_id) + .all() + ) + + @rollback_on_exc + def update_org_incident_services(self, incident_services: List[OrgIncidentService]): + [ + self._db.session.merge(incident_service) + for incident_service in incident_services + ] + self._db.session.commit() + + @rollback_on_exc + def get_incidents_bookmark( + self, + entity_id: str, + entity_type: IncidentBookmarkType, + provider: IncidentProvider, + ) -> IncidentsBookmark: + return ( + self._db.session.query(IncidentsBookmark) + .filter( + and_( + IncidentsBookmark.entity_id == entity_id, + IncidentsBookmark.entity_type == entity_type, + IncidentsBookmark.provider == provider.value, + ) + ) + .one_or_none() + ) + + @rollback_on_exc + def save_incidents_bookmark(self, bookmark: IncidentsBookmark): + self._db.session.merge(bookmark) + self._db.session.commit() + + @rollback_on_exc + def save_incidents_data( + self, + incidents: List[Incident], + incident_org_incident_service_map: List[IncidentOrgIncidentServiceMap], + ): + [self._db.session.merge(incident) for incident in incidents] + [ + self._db.session.merge(incident_service_map) + for incident_service_map in incident_org_incident_service_map + ] + self._db.session.commit() + + @rollback_on_exc + def get_resolved_team_incidents( + self, team_id: str, interval: Interval, incident_filter: IncidentFilter = None + ) -> List[Incident]: + query = self._get_team_incidents_query(team_id, incident_filter) + + query = query.filter( + and_( + Incident.status == IncidentStatus.RESOLVED.value, + Incident.resolved_date.between(interval.from_time, interval.to_time), + ) + ) + + return query.all() + + @rollback_on_exc + def get_team_incidents( + self, team_id: str, interval: Interval, incident_filter: IncidentFilter = None + ) -> List[Incident]: + query = self._get_team_incidents_query(team_id, incident_filter) + + query = query.filter( + Incident.creation_date.between(interval.from_time, interval.to_time), + ) + + return query.all() + + @rollback_on_exc + def get_incident_by_key_type_and_provider( + self, key: str, incident_type: IncidentType, provider: IncidentProvider + ) -> Incident: + return ( + self._db.session.query(Incident) + .filter( + and_( + Incident.key == key, + Incident.incident_type == incident_type, + Incident.provider == provider.value, + ) + ) + .one_or_none() + ) + + def _get_team_incidents_query( + self, team_id: str, incident_filter: IncidentFilter = None + ): + query = ( + self._db.session.query(Incident) + .join( + IncidentOrgIncidentServiceMap, + Incident.id == IncidentOrgIncidentServiceMap.incident_id, + ) + .join( + TeamIncidentService, + IncidentOrgIncidentServiceMap.service_id + == TeamIncidentService.service_id, + ) + .filter( + TeamIncidentService.team_id == team_id, + ) + ) + + query = self._apply_incident_filter(query, incident_filter) + + return query.order_by(Incident.creation_date.asc()) + + def _apply_incident_filter(self, query, incident_filter: IncidentFilter = None): + if not incident_filter: + return query + query = query.filter(*incident_filter.filter_query) + return query diff --git a/backend/analytics_server/dora/store/repos/integrations.py b/backend/analytics_server/dora/store/repos/integrations.py new file mode 100644 index 000000000..6ca35ce41 --- /dev/null +++ b/backend/analytics_server/dora/store/repos/integrations.py @@ -0,0 +1,2 @@ +class IntegrationsRepoService: + pass diff --git a/backend/analytics_server/dora/store/repos/settings.py b/backend/analytics_server/dora/store/repos/settings.py new file mode 100644 index 000000000..24736d004 --- /dev/null +++ b/backend/analytics_server/dora/store/repos/settings.py @@ -0,0 +1,90 @@ +from typing import Optional, List + +from sqlalchemy import and_ + +from dora.store import db, rollback_on_exc +from dora.store.models import ( + Settings, + SettingType, + EntityType, + Users, +) +from dora.utils.time import time_now + + +class SettingsRepoService: + def __init__(self): + self._db = db + + @rollback_on_exc + def get_setting( + self, entity_id: str, entity_type: EntityType, setting_type: SettingType + ) -> Optional[Settings]: + return ( + self._db.session.query(Settings) + .filter( + and_( + Settings.setting_type == setting_type, + Settings.entity_type == entity_type, + Settings.entity_id == entity_id, + Settings.is_deleted == False, + ) + ) + .one_or_none() + ) + + @rollback_on_exc + def create_settings(self, settings: List[Settings]) -> List[Settings]: + [self._db.session.merge(setting) for setting in settings] + self._db.session.commit() + return settings + + @rollback_on_exc + def save_setting(self, setting: Settings) -> Optional[Settings]: + self._db.session.merge(setting) + self._db.session.commit() + + return self.get_setting( + entity_id=setting.entity_id, + entity_type=setting.entity_type, + setting_type=setting.setting_type, + ) + + @rollback_on_exc + def delete_setting( + self, + entity_id: str, + entity_type: EntityType, + setting_type: SettingType, + deleted_by: Users, + ) -> Optional[Settings]: + setting = self.get_setting(entity_id, entity_type, setting_type) + if not setting: + return + + setting.is_deleted = True + setting.updated_by = deleted_by.id + setting.updated_at = time_now() + self._db.session.merge(setting) + self._db.session.commit() + return setting + + @rollback_on_exc + def get_settings( + self, + entity_id: str, + entity_type: EntityType, + setting_types: List[SettingType], + ) -> Optional[Settings]: + return ( + self._db.session.query(Settings) + .filter( + and_( + Settings.setting_type.in_(setting_types), + Settings.entity_type == entity_type, + Settings.entity_id == entity_id, + Settings.is_deleted == False, + ) + ) + .all() + ) diff --git a/backend/analytics_server/dora/store/repos/workflows.py b/backend/analytics_server/dora/store/repos/workflows.py new file mode 100644 index 000000000..8b8fefa23 --- /dev/null +++ b/backend/analytics_server/dora/store/repos/workflows.py @@ -0,0 +1,227 @@ +from datetime import datetime +from typing import List, Tuple + +from sqlalchemy.orm import defer +from sqlalchemy import and_ + +from dora.store import db, rollback_on_exc +from dora.store.models.code.workflows.enums import ( + RepoWorkflowRunsStatus, + RepoWorkflowType, + RepoWorkflowProviders, +) +from dora.store.models.code.workflows.filter import WorkflowFilter +from dora.store.models.code.workflows.workflows import ( + RepoWorkflow, + RepoWorkflowRuns, + RepoWorkflowRunsBookmark, +) +from dora.utils.time import Interval + + +class WorkflowRepoService: + def __init__(self): + self._db = db + + @rollback_on_exc + def get_active_repo_workflows_by_repo_ids_and_providers( + self, repo_ids: List[str], providers: List[RepoWorkflowProviders] + ) -> List[RepoWorkflow]: + + return ( + self._db.session.query(RepoWorkflow) + .options(defer(RepoWorkflow.meta)) + .filter( + RepoWorkflow.org_repo_id.in_(repo_ids), + RepoWorkflow.provider.in_(providers), + RepoWorkflow.is_active.is_(True), + ) + .all() + ) + + @rollback_on_exc + def get_repo_workflow_run_by_provider_workflow_run_id( + self, repo_workflow_id: str, provider_workflow_run_id: str + ) -> RepoWorkflowRuns: + return ( + self._db.session.query(RepoWorkflowRuns) + .filter( + RepoWorkflowRuns.repo_workflow_id == repo_workflow_id, + RepoWorkflowRuns.provider_workflow_run_id == provider_workflow_run_id, + ) + .one_or_none() + ) + + @rollback_on_exc + def save_repo_workflow_runs(self, repo_workflow_runs: List[RepoWorkflowRuns]): + [ + self._db.session.merge(repo_workflow_run) + for repo_workflow_run in repo_workflow_runs + ] + self._db.session.commit() + + @rollback_on_exc + def get_repo_workflow_runs_bookmark( + self, repo_workflow_id: str + ) -> RepoWorkflowRunsBookmark: + return ( + self._db.session.query(RepoWorkflowRunsBookmark) + .filter(RepoWorkflowRunsBookmark.repo_workflow_id == repo_workflow_id) + .one_or_none() + ) + + @rollback_on_exc + def update_repo_workflow_runs_bookmark(self, bookmark: RepoWorkflowRunsBookmark): + self._db.session.merge(bookmark) + self._db.session.commit() + + @rollback_on_exc + def get_repo_workflow_by_repo_ids( + self, repo_ids: List[str], type: RepoWorkflowType + ) -> List[RepoWorkflow]: + return ( + self._db.session.query(RepoWorkflow) + .options(defer(RepoWorkflow.meta)) + .filter( + and_( + RepoWorkflow.org_repo_id.in_(repo_ids), + RepoWorkflow.type == type, + RepoWorkflow.is_active.is_(True), + ) + ) + .all() + ) + + @rollback_on_exc + def get_repo_workflows_by_repo_id(self, repo_id: str) -> List[RepoWorkflow]: + return ( + self._db.session.query(RepoWorkflow) + .options(defer(RepoWorkflow.meta)) + .filter( + RepoWorkflow.org_repo_id == repo_id, + RepoWorkflow.is_active.is_(True), + ) + .all() + ) + + @rollback_on_exc + def get_successful_repo_workflows_runs_by_repo_ids( + self, repo_ids: List[str], interval: Interval, workflow_filter: WorkflowFilter + ) -> List[Tuple[RepoWorkflow, RepoWorkflowRuns]]: + query = ( + self._db.session.query(RepoWorkflow, RepoWorkflowRuns) + .options(defer(RepoWorkflow.meta), defer(RepoWorkflowRuns.meta)) + .join( + RepoWorkflowRuns, RepoWorkflow.id == RepoWorkflowRuns.repo_workflow_id + ) + ) + query = self._filter_active_repo_workflows(query) + query = self._filter_repo_workflows_by_repo_ids(query, repo_ids) + query = self._filter_repo_workflow_runs_in_interval(query, interval) + query = self._filter_repo_workflow_runs_status( + query, RepoWorkflowRunsStatus.SUCCESS + ) + + query = self._filter_workflows(query, workflow_filter) + + query = query.order_by(RepoWorkflowRuns.conducted_at.asc()) + + return query.all() + + @rollback_on_exc + def get_repos_workflow_runs_by_repo_ids( + self, + repo_ids: List[str], + interval: Interval, + workflow_filter: WorkflowFilter = None, + ) -> List[Tuple[RepoWorkflow, RepoWorkflowRuns]]: + query = ( + self._db.session.query(RepoWorkflow, RepoWorkflowRuns) + .options(defer(RepoWorkflow.meta), defer(RepoWorkflowRuns.meta)) + .join( + RepoWorkflowRuns, RepoWorkflow.id == RepoWorkflowRuns.repo_workflow_id + ) + ) + query = self._filter_active_repo_workflows(query) + query = self._filter_active_repo_workflows(query) + query = self._filter_repo_workflows_by_repo_ids(query, repo_ids) + query = self._filter_repo_workflow_runs_in_interval(query, interval) + + query = self._filter_workflows(query, workflow_filter) + + query = query.order_by(RepoWorkflowRuns.conducted_at.asc()) + + return query.all() + + @rollback_on_exc + def get_repo_workflow_run_by_id( + self, repo_workflow_run_id: str + ) -> Tuple[RepoWorkflow, RepoWorkflowRuns]: + return ( + self._db.session.query(RepoWorkflow, RepoWorkflowRuns) + .options(defer(RepoWorkflow.meta), defer(RepoWorkflowRuns.meta)) + .join(RepoWorkflow, RepoWorkflow.id == RepoWorkflowRuns.repo_workflow_id) + .filter(RepoWorkflowRuns.id == repo_workflow_run_id) + .one_or_none() + ) + + @rollback_on_exc + def get_previous_workflow_run( + self, workflow_run: RepoWorkflowRuns + ) -> Tuple[RepoWorkflow, RepoWorkflowRuns]: + return ( + self._db.session.query(RepoWorkflow, RepoWorkflowRuns) + .options(defer(RepoWorkflow.meta), defer(RepoWorkflowRuns.meta)) + .join(RepoWorkflow, RepoWorkflow.id == RepoWorkflowRuns.repo_workflow_id) + .filter( + RepoWorkflowRuns.repo_workflow_id == workflow_run.repo_workflow_id, + RepoWorkflowRuns.conducted_at < workflow_run.conducted_at, + RepoWorkflowRuns.head_branch == workflow_run.head_branch, + ) + .order_by(RepoWorkflowRuns.conducted_at.desc()) + .first() + ) + + @rollback_on_exc + def get_repo_workflow_runs_conducted_after_time( + self, repo_id: str, from_time: datetime = None, limit_value: int = 500 + ): + query = ( + self._db.session.query(RepoWorkflowRuns) + .options(defer(RepoWorkflowRuns.meta)) + .join(RepoWorkflow, RepoWorkflow.id == RepoWorkflowRuns.repo_workflow_id) + .filter( + RepoWorkflow.org_repo_id == repo_id, + RepoWorkflow.is_active.is_(True), + RepoWorkflowRuns.status == RepoWorkflowRunsStatus.SUCCESS, + ) + ) + + if from_time: + query = query.filter(RepoWorkflowRuns.conducted_at >= from_time) + + query = query.order_by(RepoWorkflowRuns.conducted_at) + + return query.limit(limit_value).all() + + def _filter_active_repo_workflows(self, query): + return query.filter( + RepoWorkflow.is_active.is_(True), + ) + + def _filter_repo_workflows_by_repo_ids(self, query, repo_ids: List[str]): + return query.filter(RepoWorkflow.org_repo_id.in_(repo_ids)) + + def _filter_repo_workflow_runs_in_interval(self, query, interval: Interval): + return query.filter( + RepoWorkflowRuns.conducted_at.between(interval.from_time, interval.to_time) + ) + + def _filter_repo_workflow_runs_status(self, query, status: RepoWorkflowRunsStatus): + return query.filter(RepoWorkflowRuns.status == status) + + def _filter_workflows(self, query, workflow_filter: WorkflowFilter): + if not workflow_filter: + return query + query = query.filter(*workflow_filter.filter_query) + return query diff --git a/backend/analytics_server/dora/utils/__init__.py b/backend/analytics_server/dora/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/dora/utils/cryptography.py b/backend/analytics_server/dora/utils/cryptography.py new file mode 100644 index 000000000..7d336b183 --- /dev/null +++ b/backend/analytics_server/dora/utils/cryptography.py @@ -0,0 +1,95 @@ +import configparser +import os +from base64 import b64encode, b64decode + +from Crypto.Cipher import PKCS1_OAEP +from Crypto.PublicKey import RSA + +service = None +CONFIG_PATH = "dora/config/config.ini" + + +class CryptoService: + def __init__(self): + self._public_key, self._public_cipher = (None, None) + self._private_key, self._private_cipher = (None, None) + + def _init_keys(self): + # Skip if already setup + if self._public_key: + return + + config = configparser.ConfigParser() + config_path = os.path.join(os.getcwd(), CONFIG_PATH) + config.read(config_path) + public_key = self._decode_key(config.get("KEYS", "SECRET_PUBLIC_KEY")) + private_key = self._decode_key(config.get("KEYS", "SECRET_PRIVATE_KEY")) + + self._public_key = RSA.importKey(public_key) if public_key else None + self._private_key = RSA.importKey(private_key) if private_key else None + + self._public_cipher = ( + PKCS1_OAEP.new(self._public_key) if self._public_key else None + ) + self._private_cipher = ( + PKCS1_OAEP.new(self._private_key) if self._private_key else None + ) + + def encrypt(self, message: str, chunk_size: int) -> [str]: + self._init_keys() + + if not message: + return message + + if not self._public_key: + raise Exception("No public key found to encrypt") + + chunks = [ + message[i : i + chunk_size] for i in range(0, len(message), chunk_size) + ] + + return [ + b64encode(self._public_cipher.encrypt(chunk.encode("utf8"))).decode("utf8") + for chunk in chunks + ] + + def decrypt(self, secret: str): + self._init_keys() + + if not secret: + return secret + + if not self._private_key: + raise Exception("No private key found to decrypt") + + secret = secret.encode("utf8") + return self._private_cipher.decrypt(b64decode(secret)).decode("utf8") + + def decrypt_chunks(self, secret_chunks: [str]): + self._init_keys() + + if not secret_chunks: + return secret_chunks + + if not self._private_key: + raise Exception("No private key found to decrypt") + + return "".join( + self._private_cipher.decrypt(b64decode(secret.encode("utf8"))).decode( + "utf8" + ) + for secret in secret_chunks + ) + + def _decode_key(self, key: str) -> str: + key = key.replace("%", "/") + key = key.encode("utf8") + return b64decode(key).decode("utf8") + + +def get_crypto_service(): + global service + if not service: + service = CryptoService() + + return service diff --git a/backend/analytics_server/dora/utils/dict.py b/backend/analytics_server/dora/utils/dict.py new file mode 100644 index 000000000..2bd91cf84 --- /dev/null +++ b/backend/analytics_server/dora/utils/dict.py @@ -0,0 +1,32 @@ +from typing import Dict, Any, List + + +def get_average_of_dict_values(key_to_int_map: Dict[any, int]) -> int: + """ + This method accepts a dictionary with any key type mapped to integer values and returns the average of those keys. Nulls are considered as zero. + """ + + if not key_to_int_map: + return 0 + + values = list(key_to_int_map.values()) + sum_of_value = 0 + for value in values: + + if value is None: + continue + + sum_of_value += value + + return sum_of_value // len(values) + + +def get_key_to_count_map_from_key_to_list_map( + week_to_list_map: Dict[Any, List[Any]] +) -> Dict[Any, int]: + """ + This method takes a dict of keys to list and returns a dict of keys mapped to the length of lists from the input dict. + """ + list_len_or_zero = lambda x: len(x) if type(x) in [list, set] else 0 + + return {key: list_len_or_zero(lst) for key, lst in week_to_list_map.items()} diff --git a/backend/analytics_server/dora/utils/github.py b/backend/analytics_server/dora/utils/github.py new file mode 100644 index 000000000..ca472d70e --- /dev/null +++ b/backend/analytics_server/dora/utils/github.py @@ -0,0 +1,50 @@ +from queue import Queue +from threading import Thread + +from github import Organization + +from dora.utils.log import LOG + + +def github_org_data_multi_thread_worker(orgs: [Organization]) -> dict: + class Worker(Thread): + def __init__(self, request_queue: Queue): + Thread.__init__(self) + self.queue = request_queue + self.results = {} + + def run(self): + while True: + if self.queue.empty(): + break + org = self.queue.get() + try: + repos = list(org.get_repos().get_page(0)[:5]) + except Exception as e: + LOG.warn(f"Error while fetching github data for {org.name}: {e}") + self.queue.task_done() + continue + self.results[org.name] = { + "repos": [repo.name for repo in repos], + } + self.queue.task_done() + + q = Queue() + num_of_workers = len(orgs) + for org in orgs: + q.put(org) + + workers = [] + for _ in range(num_of_workers): + worker = Worker(q) + worker.start() + workers.append(worker) + + for worker in workers: + worker.join() + + # Combine results from all workers + r = {} + for worker in workers: + r.update(worker.results) + return r diff --git a/backend/analytics_server/dora/utils/lock.py b/backend/analytics_server/dora/utils/lock.py new file mode 100644 index 000000000..527e1ae3d --- /dev/null +++ b/backend/analytics_server/dora/utils/lock.py @@ -0,0 +1,41 @@ +from os import getenv + +from redis import Redis +from redis_lock import Lock + +REDIS_HOST = getenv("REDIS_HOST", "localhost") +REDIS_PORT = getenv("REDIS_PORT", 6379) +REDIS_DB = 0 +REDIS_PASSWORD = "" +SSL_STATUS = True if REDIS_HOST != "localhost" else False + +service = None + + +class RedisLockService: + def __init__(self, host, port, db, password, ssl): + self.host = host + self.port = port + self.db = db + self.password = password + self.ssl = ssl + self.redis = Redis( + host=self.host, + port=self.port, + db=self.db, + password=self.password, + ssl=self.ssl, + socket_connect_timeout=5, + ) + + def acquire_lock(self, key: str): + return Lock(self.redis, name=key, expire=1.5, auto_renewal=True) + + +def get_redis_lock_service(): + global service + if not service: + service = RedisLockService( + REDIS_HOST, REDIS_PORT, REDIS_DB, REDIS_PASSWORD, SSL_STATUS + ) + return service diff --git a/backend/analytics_server/dora/utils/log.py b/backend/analytics_server/dora/utils/log.py new file mode 100644 index 000000000..c7c8882e3 --- /dev/null +++ b/backend/analytics_server/dora/utils/log.py @@ -0,0 +1,17 @@ +import logging + +LOG = logging.getLogger() + + +def custom_logging(func): + def wrapper(*args, **kwargs): + print( + f"[{func.__name__.upper()}]", args[0] + ) # Assuming the first argument is the log message + return func(*args, **kwargs) + + return wrapper + + +LOG.error = custom_logging(LOG.error) +LOG.info = custom_logging(LOG.info) diff --git a/backend/analytics_server/dora/utils/regex.py b/backend/analytics_server/dora/utils/regex.py new file mode 100644 index 000000000..b0893ee60 --- /dev/null +++ b/backend/analytics_server/dora/utils/regex.py @@ -0,0 +1,29 @@ +import re +from typing import List +from werkzeug.exceptions import BadRequest + + +def check_regex(pattern: str): + # pattern is a string containing the regex pattern + try: + re.compile(pattern) + + except re.error: + return False + + return True + + +def check_all_regex(patterns: List[str]) -> bool: + # patterns is a list of strings containing the regex patterns + for pattern in patterns: + if not pattern or not check_regex(pattern): + return False + + return True + + +def regex_list(patterns: List[str]) -> List[str]: + if not check_all_regex(patterns): + raise BadRequest("Invalid regex pattern") + return patterns diff --git a/backend/analytics_server/dora/utils/string.py b/backend/analytics_server/dora/utils/string.py new file mode 100644 index 000000000..a56caf067 --- /dev/null +++ b/backend/analytics_server/dora/utils/string.py @@ -0,0 +1,5 @@ +from uuid import uuid4 + + +def uuid4_str(): + return str(uuid4()) diff --git a/backend/analytics_server/dora/utils/time.py b/backend/analytics_server/dora/utils/time.py new file mode 100644 index 000000000..6e8b7ceb8 --- /dev/null +++ b/backend/analytics_server/dora/utils/time.py @@ -0,0 +1,270 @@ +from datetime import datetime, timedelta +from typing import Callable, List, Dict, Any, Optional +from collections import defaultdict + +import pytz + +ISO_8601_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" + + +def time_now(): + return datetime.now().astimezone(pytz.UTC) + + +class Interval: + def __init__(self, from_time: datetime, to_time: datetime): + assert ( + to_time >= from_time + ), f"from_time: {from_time.isoformat()} is greater than to_time: {to_time.isoformat()}" + self._from_time = from_time + self._to_time = to_time + + def __contains__(self, dt: datetime): + return self.from_time < dt < self.to_time + + @property + def from_time(self): + return self._from_time + + @property + def to_time(self): + return self._to_time + + @property + def duration(self) -> timedelta: + return self._to_time - self._from_time + + def overlaps(self, interval): + if interval.from_time <= self.to_time < interval.to_time: + return True + + if self.from_time <= interval.from_time < self.to_time: + return True + + return False + + def merge(self, interval): + return Interval( + min(self.from_time, interval.from_time), max(self.to_time, interval.to_time) + ) + + def merge(self, interval: []): + return Interval( + min(self.from_time, interval.from_time), max(self.to_time, interval.to_time) + ) + + def __str__(self): + return f"{self._from_time.isoformat()} -> {self._to_time.isoformat()}" + + def __repr__(self): + return str(self) + + def __eq__(self, other): + return self.from_time == other.from_time and self.to_time == other.to_time + + @staticmethod + def merge_intervals(intervals): + if not intervals or len(intervals) == 1: + return intervals + + intervals.sort(key=lambda x: (x.from_time, x.to_time)) + merged_intervals = [intervals[0]] + for interval in intervals[1:]: + if merged_intervals[-1].overlaps(interval): + merged_intervals[-1] = merged_intervals[-1].merge(interval) + else: + merged_intervals.append(interval) + + return merged_intervals + + def get_remaining_intervals(self, intervals): + if not intervals: + return [self] + + intervals = Interval.merge_intervals(intervals) + + free_intervals = [] + fro, to = self.from_time, self.to_time + for interval in intervals: + if interval.from_time > fro: + free_intervals.append(Interval(fro, interval.from_time)) + fro = interval.to_time + + if fro < to: + free_intervals.append(Interval(fro, to)) + + return free_intervals + + +def get_start_of_day(date: datetime) -> datetime: + return datetime(date.year, date.month, date.day, 0, 0, 0, tzinfo=pytz.UTC) + + +def get_end_of_day(date: datetime) -> datetime: + return datetime( + date.year, date.month, date.day, 23, 59, 59, 999999, tzinfo=pytz.UTC + ) + + +def get_given_weeks_monday(dt: datetime): + monday = dt - timedelta(days=dt.weekday()) + + monday_midnight = datetime( + monday.year, monday.month, monday.day, 0, 0, 0, tzinfo=pytz.UTC + ) + + return monday_midnight + + +def get_given_weeks_sunday(dt: datetime): + sunday = dt + timedelta(days=(6 - dt.weekday())) + sunday_midnight = datetime(sunday.year, sunday.month, sunday.day, tzinfo=pytz.UTC) + return get_end_of_day(sunday_midnight) + + +def get_time_delta_based_on_granularity(date: datetime, granularity: str) -> timedelta: + """ + Takes a date and a granularity. + Returns a timedelta based on the granularity. + Granularity options: 'daily', 'weekly', 'monthly'. + """ + if granularity == "daily": + return timedelta(days=1) + if granularity == "weekly": + return timedelta(weeks=1) + if granularity == "monthly": + some_day_in_next_month = date.replace(day=28) + timedelta(days=4) + last_day_of_month = some_day_in_next_month - timedelta( + days=some_day_in_next_month.day + ) + return last_day_of_month - date + timedelta(days=1) + raise ValueError("Invalid granularity. Choose 'daily', 'weekly', or 'monthly'.") + + +def get_expanded_interval_based_on_granularity( + interval: Interval, granularity: str +) -> Interval: + """ + Takes an interval and a granularity. + Returns an expanded interval based on the granularity. + Granularity options: 'daily', 'weekly', 'monthly'. + """ + if granularity == "daily": + return Interval( + get_start_of_day(interval.from_time), get_end_of_day(interval.to_time) + ) + if granularity == "weekly": + return Interval( + get_given_weeks_monday(interval.from_time), + get_given_weeks_sunday(interval.to_time), + ) + if granularity == "monthly": + some_day_in_next_month = interval.to_time.replace(day=28) + timedelta(days=4) + return Interval( + datetime( + interval.from_time.year, interval.from_time.month, 1, tzinfo=pytz.UTC + ), + get_end_of_day( + some_day_in_next_month - timedelta(days=some_day_in_next_month.day) + ), + ) + raise ValueError("Invalid granularity. Choose 'daily', 'weekly', or 'monthly'.") + + +def generate_expanded_buckets( + lst: List[Any], + interval: Interval, + datetime_attribute: str, + granularity: str = "weekly", +) -> Dict[datetime, List[Any]]: + """ + Takes a list of objects, time interval, a datetime_attribute string, and a granularity. + Buckets the list of objects based on the specified granularity of the datetime_attribute. + The series is expanded beyond the input interval based on the datetime_attribute. + Granularity options: 'daily', 'weekly', 'monthly'. + """ + from_time = interval.from_time + to_time = interval.to_time + + def generate_empty_buckets( + from_time: datetime, to_time: datetime, granularity: str + ) -> Dict[datetime, List[Any]]: + buckets_map: Dict[datetime, List[Any]] = defaultdict(list) + expanded_interval = get_expanded_interval_based_on_granularity( + Interval(from_time, to_time), granularity + ) + curr_date = expanded_interval.from_time + while curr_date <= expanded_interval.to_time: + delta = get_time_delta_based_on_granularity(curr_date, granularity) + buckets_map[get_start_of_day(curr_date)] = [] + curr_date += delta + + return buckets_map + + for obj in lst: + if not isinstance(getattr(obj, datetime_attribute), datetime): + raise ValueError( + f"Type of datetime_attribute {type(getattr(obj, datetime_attribute))} is not datetime" + ) + + buckets_map: Dict[datetime, List[Any]] = generate_empty_buckets( + from_time, to_time, granularity + ) + + for obj in lst: + date_value = getattr(obj, datetime_attribute) + if granularity == "daily": + bucket_key = get_start_of_day(date_value) + elif granularity == "weekly": + # Adjust the date to the start of the week (Monday) + bucket_key = get_start_of_day( + date_value - timedelta(days=date_value.weekday()) + ) + elif granularity == "monthly": + # Adjust the date to the start of the month + bucket_key = get_start_of_day(date_value.replace(day=1)) + else: + raise ValueError( + "Invalid granularity. Choose 'daily', 'weekly', or 'monthly'." + ) + + buckets_map[bucket_key].append(obj) + + return buckets_map + + +def sort_dict_by_datetime_keys(input_dict): + sorted_items = sorted(input_dict.items()) + sorted_dict = dict(sorted_items) + return sorted_dict + + +def fill_missing_week_buckets( + week_start_to_object_map: Dict[datetime, Any], + interval: Interval, + callable_class: Optional[Callable] = None, +) -> Dict[datetime, Any]: + """ + Takes a dict of week_start to object map. + Add the missing weeks with default value of the class/callable. + If no callable is passed, the missing weeks are set to None. + """ + first_monday = get_given_weeks_monday(interval.from_time) + last_sunday = get_given_weeks_sunday(interval.to_time) + + curr_day = first_monday + week_start_to_object_map_with_weeks_in_interval = {} + + while curr_day < last_sunday: + if curr_day not in week_start_to_object_map: + week_start_to_object_map_with_weeks_in_interval[curr_day] = ( + callable_class() if callable_class else None + ) + else: + week_start_to_object_map_with_weeks_in_interval[ + curr_day + ] = week_start_to_object_map[curr_day] + + curr_day = curr_day + timedelta(days=7) + + return sort_dict_by_datetime_keys(week_start_to_object_map_with_weeks_in_interval) diff --git a/apiserver/env.py b/backend/analytics_server/env.py similarity index 64% rename from apiserver/env.py rename to backend/analytics_server/env.py index 815f0284b..f07f2a857 100644 --- a/apiserver/env.py +++ b/backend/analytics_server/env.py @@ -5,6 +5,6 @@ def load_app_env(): if getenv("FLASK_ENV") == "production": - load_dotenv(".env.prod") + load_dotenv("../.env.prod") else: - load_dotenv(".env.local") + load_dotenv("../.env.local") diff --git a/backend/analytics_server/tests/__init__.py b/backend/analytics_server/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/factories/__init__.py b/backend/analytics_server/tests/factories/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/factories/models/__init__.py b/backend/analytics_server/tests/factories/models/__init__.py new file mode 100644 index 000000000..9d80b2096 --- /dev/null +++ b/backend/analytics_server/tests/factories/models/__init__.py @@ -0,0 +1,12 @@ +from .code import ( + get_repo_workflow_run, + get_deployment, + get_pull_request, + get_pull_request_commit, + get_pull_request_event, +) +from .incidents import ( + get_incident, + get_change_failure_rate_metrics, + get_org_incident_service, +) diff --git a/backend/analytics_server/tests/factories/models/code.py b/backend/analytics_server/tests/factories/models/code.py new file mode 100644 index 000000000..51adc2e6d --- /dev/null +++ b/backend/analytics_server/tests/factories/models/code.py @@ -0,0 +1,201 @@ +from random import randint +from uuid import uuid4 +from dora.service.deployments.models.models import ( + Deployment, + DeploymentFrequencyMetrics, + DeploymentStatus, + DeploymentType, +) +from dora.utils.string import uuid4_str + +from dora.store.models.code import ( + PullRequestCommit, + PullRequestEvent, + PullRequestEventType, + PullRequestState, + PullRequest, + RepoWorkflowRuns, + RepoWorkflowRunsStatus, +) +from dora.utils.time import time_now + + +def get_pull_request( + id=None, + repo_id=None, + number=None, + author=None, + state=None, + title=None, + head_branch=None, + base_branch=None, + provider=None, + requested_reviews=None, + data=None, + state_changed_at=None, + created_at=None, + updated_at=None, + meta=None, + reviewers=None, + first_commit_to_open=None, + first_response_time=None, + rework_time=None, + merge_time=None, + cycle_time=None, + merge_to_deploy=None, + lead_time=None, + url=None, + merge_commit_sha=None, +): + return PullRequest( + id=id or uuid4(), + repo_id=repo_id or uuid4(), + number=number or randint(10, 100), + author=author or "randomuser", + title=title or "title", + state=state or PullRequestState.OPEN, + head_branch=head_branch or "feature", + base_branch=base_branch or "main", + provider=provider or "github", + requested_reviews=requested_reviews or [], + data=data or {}, + state_changed_at=state_changed_at or time_now(), + created_at=created_at or time_now(), + updated_at=updated_at or time_now(), + first_commit_to_open=first_commit_to_open, + first_response_time=first_response_time, + rework_time=rework_time, + merge_time=merge_time, + cycle_time=cycle_time, + merge_to_deploy=merge_to_deploy, + lead_time=lead_time, + reviewers=reviewers + if reviewers is not None + else ["randomuser1", "randomuser2"], + meta=meta or {}, + url=url, + merge_commit_sha=merge_commit_sha, + ) + + +def get_pull_request_event( + id=None, + pull_request_id=None, + type=None, + reviewer=None, + state=None, + created_at=None, + idempotency_key=None, + org_repo_id=None, + data=None, +): + return PullRequestEvent( + id=id or uuid4(), + pull_request_id=pull_request_id or uuid4(), + type=type or PullRequestEventType.REVIEW.value, + data={ + "user": {"login": reviewer or "User"}, + "state": state or "APPROVED", + "author_association": "NONE", + } + if not data + else data, + created_at=created_at or time_now(), + idempotency_key=idempotency_key or str(randint(10, 100)), + org_repo_id=org_repo_id or uuid4(), + actor_username=reviewer or "randomuser", + ) + + +def get_pull_request_commit( + hash=None, + pr_id=None, + message=None, + url=None, + data=None, + author=None, + created_at=None, + org_repo_id=None, +): + return PullRequestCommit( + hash=hash or uuid4(), + pull_request_id=pr_id or uuid4(), + message=message or "message", + url=url or "https://abc.com", + data=data or dict(), + author=author or "randomuser", + created_at=created_at or time_now(), + org_repo_id=org_repo_id or uuid4(), + ) + + +def get_repo_workflow_run( + id=None, + repo_workflow_id=None, + provider_workflow_run_id=None, + event_actor=None, + head_branch=None, + status=None, + conducted_at=None, + created_at=None, + updated_at=None, + meta=None, + duration=None, + html_url=None, +): + return RepoWorkflowRuns( + id=id or uuid4(), + repo_workflow_id=repo_workflow_id or uuid4(), + provider_workflow_run_id=provider_workflow_run_id or "1234567", + event_actor=event_actor or "samad-yar-khan", + head_branch=head_branch or "master", + status=status or RepoWorkflowRunsStatus.SUCCESS, + conducted_at=conducted_at or time_now(), + created_at=created_at or time_now(), + updated_at=updated_at or time_now(), + duration=duration, + meta=meta, + html_url=html_url, + ) + + +def get_deployment( + repo_id=None, + entity_id=None, + actor=None, + head_branch=None, + status=None, + conducted_at=None, + meta=None, + duration=None, + html_url=None, + provider=None, +): + return Deployment( + deployment_type=DeploymentType.WORKFLOW, + repo_id=repo_id or "1234567", + entity_id=entity_id or uuid4_str(), + provider=provider or "github", + actor=actor or "samad-yar-khan", + head_branch=head_branch or "master", + conducted_at=conducted_at or time_now(), + duration=duration, + status=status or DeploymentStatus.SUCCESS, + html_url=html_url or "", + meta=meta or {}, + ) + + +def get_deployment_frequency_metrics( + total_deployments=0, + daily_deployment_frequency=0, + avg_weekly_deployment_frequency=0, + avg_monthly_deployment_frequency=0, +) -> DeploymentFrequencyMetrics: + + return DeploymentFrequencyMetrics( + total_deployments=total_deployments or 0, + daily_deployment_frequency=daily_deployment_frequency or 0, + avg_weekly_deployment_frequency=avg_weekly_deployment_frequency or 0, + avg_monthly_deployment_frequency=avg_monthly_deployment_frequency or 0, + ) diff --git a/backend/analytics_server/tests/factories/models/exapi/__init__.py b/backend/analytics_server/tests/factories/models/exapi/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/factories/models/exapi/github.py b/backend/analytics_server/tests/factories/models/exapi/github.py new file mode 100644 index 000000000..8dd11a868 --- /dev/null +++ b/backend/analytics_server/tests/factories/models/exapi/github.py @@ -0,0 +1,159 @@ +from collections import namedtuple +from dataclasses import dataclass +from datetime import datetime +from typing import Dict + +from dora.utils.time import time_now + + +def get_github_commit_dict( + sha: str = "123456789098765", + author_login: str = "author_abc", + url: str = "https://github.com/123456789098765", + message: str = "[abc 315] avoid mapping edit state", + created_at: str = "2022-06-29T10:53:15Z", +) -> Dict: + return { + "sha": sha, + "commit": { + "committer": {"name": "abc", "email": "abc@midd.com", "date": created_at}, + "message": message, + }, + "author": { + "login": author_login, + "id": 95607047, + "node_id": "abc", + "avatar_url": "", + }, + "html_url": url, + } + + +@dataclass +class GithubPullRequestReview: + id: str + submitted_at: datetime + user_login: str + + @property + def raw_data(self): + return { + "id": self.id, + "submitted_at": self.submitted_at, + "user": { + "login": self.user_login, + }, + } + + +def get_github_pull_request_review( + review_id: str = "123456", + submitted_at: datetime = time_now(), + user_login: str = "abc", +) -> GithubPullRequestReview: + + return GithubPullRequestReview(review_id, submitted_at, user_login) + + +Branch = namedtuple("Branch", ["ref"]) +User = namedtuple("User", ["login"]) + + +@dataclass +class GithubPullRequest: + number: int + merged_at: datetime + closed_at: datetime + title: str + html_url: str + created_at: datetime + updated_at: datetime + base: Branch + head: Branch + user: User + commits: int + additions: int + deletions: int + changed_files: int + merge_commit_sha: str + + @property + def raw_data(self): + return { + "number": self.number, + "merged_at": self.merged_at, + "closed_at": self.closed_at, + "title": self.title, + "html_url": self.html_url, + "created_at": self.created_at, + "updated_at": self.updated_at, + "base": {"ref": self.base.ref}, + "head": {"ref": self.head.ref}, + "user": {"login": self.user.login}, + "commits": self.commits, + "additions": self.additions, + "deletions": self.deletions, + "changed_files": self.changed_files, + "requested_reviewers": [], + "merge_commit_sha": self.merge_commit_sha, + } + + +def get_github_pull_request( + number: int = 1, + merged_at: datetime = None, + closed_at: datetime = None, + title: str = "random_title", + html_url: str = None, + created_at: datetime = time_now(), + updated_at: datetime = time_now(), + base_ref: str = "main", + head_ref: str = "feature", + user_login: str = "abc", + commits: int = 1, + additions: int = 1, + deletions: int = 1, + changed_files: int = 1, + merge_commit_sha: str = "123456", +) -> GithubPullRequest: + return GithubPullRequest( + number, + merged_at, + closed_at, + title, + html_url, + created_at, + updated_at, + Branch(base_ref), + Branch(head_ref), + User(user_login), + commits, + additions, + deletions, + changed_files, + merge_commit_sha, + ) + + +def get_github_workflow_run_dict( + run_id: str = "123456", + actor_login: str = "abc", + head_branch: str = "feature", + status: str = "completed", + conclusion: str = "success", + run_started_at: str = "2022-06-29T10:53:15Z", + created_at: str = "2022-06-29T10:53:15Z", + updated_at: str = "2022-06-29T10:53:15Z", + html_url: str = "", +) -> Dict: + return { + "id": run_id, + "actor": {"login": actor_login}, + "head_branch": head_branch, + "status": status, + "conclusion": conclusion, + "run_started_at": run_started_at, + "created_at": created_at, + "updated_at": updated_at, + "html_url": html_url, + } diff --git a/backend/analytics_server/tests/factories/models/incidents.py b/backend/analytics_server/tests/factories/models/incidents.py new file mode 100644 index 000000000..bf0c3af42 --- /dev/null +++ b/backend/analytics_server/tests/factories/models/incidents.py @@ -0,0 +1,93 @@ +from datetime import datetime +from typing import List, Set + +from voluptuous import default_factory +from dora.service.deployments.models.models import Deployment +from dora.service.incidents.models.mean_time_to_recovery import ChangeFailureRateMetrics + +from dora.store.models.incidents import IncidentType, OrgIncidentService +from dora.store.models.incidents.incidents import ( + Incident, + IncidentOrgIncidentServiceMap, +) +from dora.utils.string import uuid4_str +from dora.utils.time import time_now + + +def get_incident( + id: str = uuid4_str(), + provider: str = "provider", + key: str = "key", + title: str = "title", + status: str = "status", + incident_number: int = 0, + incident_type: IncidentType = IncidentType("INCIDENT"), + creation_date: datetime = time_now(), + created_at: datetime = time_now(), + updated_at: datetime = time_now(), + resolved_date: datetime = time_now(), + acknowledged_date: datetime = time_now(), + assigned_to: str = "assigned_to", + assignees: List[str] = default_factory(list), + meta: dict = default_factory(dict), +) -> Incident: + return Incident( + id=id, + provider=provider, + key=key, + title=title, + status=status, + incident_number=incident_number, + incident_type=incident_type, + created_at=created_at, + updated_at=updated_at, + creation_date=creation_date, + resolved_date=resolved_date, + assigned_to=assigned_to, + assignees=assignees, + acknowledged_date=acknowledged_date, + meta=meta, + ) + + +def get_org_incident_service( + service_id: str, + org_id: str = uuid4_str(), + name: str = "Service", + provider: str = "PagerDuty", + key: str = "service_key", + auto_resolve_timeout: int = 0, + acknowledgement_timeout: int = 0, + created_by: str = "user", + provider_team_keys=default_factory(list), + status: str = "active", + meta: dict = default_factory(dict), +): + return OrgIncidentService( + id=service_id if service_id else uuid4_str(), + org_id=org_id if org_id else uuid4_str(), + name=name, + provider=provider, + key=key, + auto_resolve_timeout=auto_resolve_timeout, + acknowledgement_timeout=acknowledgement_timeout, + created_by=created_by, + provider_team_keys=provider_team_keys, + status=status, + meta=meta, + created_at=time_now(), + updated_at=time_now(), + ) + + +def get_incident_org_incident_map( + incident_id: str = uuid4_str(), service_id: str = uuid4_str() +): + return IncidentOrgIncidentServiceMap(incident_id=incident_id, service_id=service_id) + + +def get_change_failure_rate_metrics( + failed_deployments: Set[Deployment] = None, + total_deployments: Set[Deployment] = None, +): + return ChangeFailureRateMetrics(failed_deployments, total_deployments) diff --git a/backend/analytics_server/tests/service/Incidents/sync/__init__.py b/backend/analytics_server/tests/service/Incidents/sync/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/service/Incidents/sync/test_etl_git_incidents_handler.py b/backend/analytics_server/tests/service/Incidents/sync/test_etl_git_incidents_handler.py new file mode 100644 index 000000000..a8aab0252 --- /dev/null +++ b/backend/analytics_server/tests/service/Incidents/sync/test_etl_git_incidents_handler.py @@ -0,0 +1,141 @@ +from dora.exapi.models.git_incidents import RevertPRMap +from dora.service.incidents.sync.etl_git_incidents_handler import GitIncidentsETLHandler +from dora.store.models.incidents import IncidentType, IncidentStatus +from dora.utils.string import uuid4_str +from dora.utils.time import time_now +from tests.factories.models import get_incident +from tests.factories.models.code import get_pull_request +from tests.factories.models.incidents import ( + get_org_incident_service, + get_incident_org_incident_map, +) +from tests.utilities import compare_objects_as_dicts + +org_id = uuid4_str() +repo_id = uuid4_str() +provider = "github" + +original_pr = get_pull_request( + id=uuid4_str(), + title="Testing PR", + repo_id=repo_id, + head_branch="feature", + base_branch="main", + provider=provider, +) + +revert_pr = get_pull_request( + id=uuid4_str(), + title='Revert "Testing PR"', + repo_id=repo_id, + head_branch="revert-feature", + base_branch="main", + provider=provider, +) + +expected_git_incident = get_incident( + id=uuid4_str(), + provider=provider, + key=str(original_pr.id), + title=original_pr.title, + incident_number=int(original_pr.number), + status=IncidentStatus.RESOLVED.value, + creation_date=original_pr.state_changed_at, + acknowledged_date=revert_pr.created_at, + resolved_date=revert_pr.state_changed_at, + assigned_to=revert_pr.author, + assignees=[revert_pr.author], + meta={ + "revert_pr": GitIncidentsETLHandler._adapt_pr_to_json(revert_pr), + "original_pr": GitIncidentsETLHandler._adapt_pr_to_json(original_pr), + "created_at": revert_pr.created_at.isoformat(), + "updated_at": revert_pr.updated_at.isoformat(), + }, + created_at=time_now(), + updated_at=time_now(), + incident_type=IncidentType.REVERT_PR, +) + + +def test_process_revert_pr_incident_given_existing_incident_map_returns_same_incident(): + class FakeIncidentsRepoService: + def get_incident_by_key_type_and_provider( + self, + *args, + **kwargs, + ): + return expected_git_incident + + git_incident_service = GitIncidentsETLHandler( + org_id, None, FakeIncidentsRepoService() + ) + + org_incident_service = get_org_incident_service( + provider="github", service_id=repo_id + ) + + revert_pr_map = RevertPRMap( + original_pr=original_pr, + revert_pr=revert_pr, + created_at=time_now(), + updated_at=time_now(), + ) + + incident, incident_service_map = git_incident_service._process_revert_pr_incident( + org_incident_service, revert_pr_map + ) + + expected_incident_org_incident_service_map = get_incident_org_incident_map( + expected_git_incident.id, service_id=repo_id + ) + + assert compare_objects_as_dicts( + expected_git_incident, incident, ["created_at", "updated_at"] + ) + + assert compare_objects_as_dicts( + expected_incident_org_incident_service_map, incident_service_map + ) + + +def test_process_revert_pr_incident_given_no_existing_incident_map_returns_new_incident(): + class FakeIncidentsRepoService: + def get_incident_by_key_type_and_provider( + self, + *args, + **kwargs, + ): + return None + + git_incident_service = GitIncidentsETLHandler( + org_id, None, FakeIncidentsRepoService() + ) + + org_incident_service = get_org_incident_service( + provider="github", service_id=repo_id + ) + + revert_pr_map = RevertPRMap( + original_pr=original_pr, + revert_pr=revert_pr, + created_at=time_now(), + updated_at=time_now(), + ) + + incident, incident_service_map = git_incident_service._process_revert_pr_incident( + org_incident_service, revert_pr_map + ) + + assert compare_objects_as_dicts( + expected_git_incident, incident, ["id", "created_at", "updated_at"] + ) + + expected_incident_org_incident_service_map = get_incident_org_incident_map( + uuid4_str(), service_id=repo_id + ) + + assert compare_objects_as_dicts( + expected_incident_org_incident_service_map, + incident_service_map, + ["incident_id"], + ) diff --git a/backend/analytics_server/tests/service/Incidents/test_change_failure_rate.py b/backend/analytics_server/tests/service/Incidents/test_change_failure_rate.py new file mode 100644 index 000000000..2f5e80945 --- /dev/null +++ b/backend/analytics_server/tests/service/Incidents/test_change_failure_rate.py @@ -0,0 +1,352 @@ +import pytz +from datetime import datetime +from datetime import timedelta +from tests.factories.models.incidents import get_change_failure_rate_metrics +from dora.service.incidents.incidents import get_incident_service +from dora.utils.time import Interval, time_now + +from tests.factories.models import get_incident, get_deployment + + +# No incidents, no deployments +def test_get_change_failure_rate_for_no_incidents_no_deployments(): + incident_service = get_incident_service() + incidents = [] + deployments = [] + change_failure_rate = incident_service.get_change_failure_rate_metrics( + deployments, + incidents, + ) + assert change_failure_rate == get_change_failure_rate_metrics([], []) + assert change_failure_rate.change_failure_rate == 0 + + +# No incidents, some deployments +def test_get_change_failure_rate_for_no_incidents_and_some_deployments(): + incident_service = get_incident_service() + incidents = [] + + deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=2)) + deployment_2 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) + + deployments = [ + deployment_1, + deployment_2, + ] + change_failure_rate = incident_service.get_change_failure_rate_metrics( + deployments, + incidents, + ) + assert change_failure_rate == get_change_failure_rate_metrics( + set(), set([deployment_2, deployment_1]) + ) + assert change_failure_rate.change_failure_rate == 0 + + +# Some incidents, no deployments +def test_get_deployment_incidents_count_map_returns_empty_dict_when_given_some_incidents_no_deployments(): + incident_service = get_incident_service() + incidents = [get_incident(creation_date=time_now() - timedelta(days=3))] + deployments = [] + change_failure_rate = incident_service.get_change_failure_rate_metrics( + deployments, + incidents, + ) + assert change_failure_rate == get_change_failure_rate_metrics(set(), set()) + assert change_failure_rate.change_failure_rate == 0 + + +# One incident between two deployments +def test_get_change_failure_rate_for_one_incidents_bw_two_deployments(): + incident_service = get_incident_service() + incidents = [get_incident(creation_date=time_now() - timedelta(days=1))] + + deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=2)) + deployment_2 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) + + deployments = [ + deployment_1, + deployment_2, + ] + + change_failure_rate = incident_service.get_change_failure_rate_metrics( + deployments, + incidents, + ) + assert change_failure_rate == get_change_failure_rate_metrics( + set([deployment_1]), set([deployment_2, deployment_1]) + ) + assert change_failure_rate.change_failure_rate == 50 + + +# One incident before two deployments +def test_get_change_failure_rate_for_one_incidents_bef_two_deployments(): + incident_service = get_incident_service() + incidents = [get_incident(creation_date=time_now() - timedelta(days=3))] + + deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=2)) + deployment_2 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) + + deployments = [ + deployment_1, + deployment_2, + ] + + change_failure_rate = incident_service.get_change_failure_rate_metrics( + deployments, + incidents, + ) + assert change_failure_rate == get_change_failure_rate_metrics( + set([]), set([deployment_2, deployment_1]) + ) + assert change_failure_rate.change_failure_rate == 0 + + +# One incident after two deployments +def test_get_change_failure_rate_for_one_incidents_after_two_deployments(): + incident_service = get_incident_service() + incidents = [get_incident(creation_date=time_now() - timedelta(hours=1))] + deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=2)) + deployment_2 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) + + deployments = [ + deployment_1, + deployment_2, + ] + + change_failure_rate = incident_service.get_change_failure_rate_metrics( + deployments, + incidents, + ) + assert change_failure_rate == get_change_failure_rate_metrics( + set([deployment_2]), set([deployment_2, deployment_1]) + ) + assert change_failure_rate.change_failure_rate == 50 + + +# Multiple incidents and deployments +def test_get_change_failure_rate_for_multi_incidents_multi_deployments(): + + incident_service = get_incident_service() + + incident_0 = get_incident(creation_date=time_now() - timedelta(days=10)) + + deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=7)) + + deployment_2 = get_deployment(conducted_at=time_now() - timedelta(days=6)) + incident_1 = get_incident(creation_date=time_now() - timedelta(days=5)) + + deployment_3 = get_deployment(conducted_at=time_now() - timedelta(days=4)) + incident_2 = get_incident(creation_date=time_now() - timedelta(days=3)) + + deployment_4 = get_deployment(conducted_at=time_now() - timedelta(days=2)) + incident_3 = get_incident(creation_date=time_now() - timedelta(hours=20)) + + deployment_5 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) + incident_4 = get_incident(creation_date=time_now() - timedelta(hours=4)) + incident_5 = get_incident(creation_date=time_now() - timedelta(hours=2)) + incident_6 = get_incident(creation_date=time_now() - timedelta(hours=1)) + + deployment_6 = get_deployment(conducted_at=time_now() - timedelta(minutes=30)) + + incidents = [ + incident_0, + incident_1, + incident_2, + incident_3, + incident_4, + incident_5, + incident_6, + ] + + deployments = [ + deployment_1, + deployment_2, + deployment_3, + deployment_4, + deployment_5, + deployment_6, + ] + + change_failure_rate = incident_service.get_change_failure_rate_metrics( + deployments, + incidents, + ) + + assert change_failure_rate == get_change_failure_rate_metrics( + set([deployment_2, deployment_3, deployment_4, deployment_5]), + set( + [ + deployment_1, + deployment_2, + deployment_3, + deployment_4, + deployment_5, + deployment_6, + ] + ), + ) + assert change_failure_rate.change_failure_rate == (4 / 6 * 100) + + +# No Incidents and Deployments +def test_get_weekly_change_failure_rate_for_no_incidents_no_deployments(): + + first_week_2024 = datetime(2024, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) + second_week_2024 = datetime(2024, 1, 8, 0, 0, 0, tzinfo=pytz.UTC) + third_week_2024 = datetime(2024, 1, 15, 0, 0, 0, tzinfo=pytz.UTC) + + from_time = first_week_2024 + timedelta(days=1) + to_time = third_week_2024 + timedelta(days=2) + + incidents = [] + deployments = [] + + incident_service = get_incident_service() + weekly_change_failure_rate = incident_service.get_weekly_change_failure_rate( + Interval(from_time, to_time), + deployments, + incidents, + ) + assert weekly_change_failure_rate == { + first_week_2024: get_change_failure_rate_metrics([], []), + second_week_2024: get_change_failure_rate_metrics([], []), + third_week_2024: get_change_failure_rate_metrics([], []), + } + + +# No Incidents and Deployments +def test_get_weekly_change_failure_rate_for_no_incidents_and_some_deployments(): + + first_week_2024 = datetime(2024, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) + second_week_2024 = datetime(2024, 1, 8, 0, 0, 0, tzinfo=pytz.UTC) + third_week_2024 = datetime(2024, 1, 15, 0, 0, 0, tzinfo=pytz.UTC) + + from_time = first_week_2024 + timedelta(days=1) + to_time = third_week_2024 + timedelta(days=2) + + deployment_1 = get_deployment(conducted_at=from_time + timedelta(days=2)) + deployment_2 = get_deployment(conducted_at=second_week_2024 + timedelta(days=2)) + deployment_3 = get_deployment(conducted_at=to_time - timedelta(hours=2)) + + deployments = [deployment_1, deployment_2, deployment_3] + + incidents = [] + + incident_service = get_incident_service() + weekly_change_failure_rate = incident_service.get_weekly_change_failure_rate( + Interval(from_time, to_time), + deployments, + incidents, + ) + + assert weekly_change_failure_rate == { + first_week_2024: get_change_failure_rate_metrics([], set([deployment_1])), + second_week_2024: get_change_failure_rate_metrics([], set([deployment_2])), + third_week_2024: get_change_failure_rate_metrics([], set([deployment_3])), + } + + +# No Incidents and Deployments +def test_get_weekly_change_failure_rate_for_incidents_and_no_deployments(): + + first_week_2024 = datetime(2024, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) + second_week_2024 = datetime(2024, 1, 8, 0, 0, 0, tzinfo=pytz.UTC) + third_week_2024 = datetime(2024, 1, 15, 0, 0, 0, tzinfo=pytz.UTC) + + from_time = first_week_2024 + timedelta(days=1) + to_time = third_week_2024 + timedelta(days=2) + + incident_1 = get_incident(creation_date=to_time - timedelta(days=14)) + incident_2 = get_incident(creation_date=to_time - timedelta(days=10)) + incident_3 = get_incident(creation_date=to_time - timedelta(days=7)) + incident_4 = get_incident(creation_date=to_time - timedelta(days=3)) + incident_5 = get_incident(creation_date=to_time - timedelta(hours=2)) + incident_6 = get_incident(creation_date=to_time - timedelta(hours=1)) + + incidents = [incident_1, incident_2, incident_3, incident_4, incident_5, incident_6] + deployments = [] + + incident_service = get_incident_service() + weekly_change_failure_rate = incident_service.get_weekly_change_failure_rate( + Interval(from_time, to_time), + deployments, + incidents, + ) + + assert weekly_change_failure_rate == { + first_week_2024: get_change_failure_rate_metrics([], []), + second_week_2024: get_change_failure_rate_metrics([], []), + third_week_2024: get_change_failure_rate_metrics([], []), + } + + +def test_get_weekly_change_failure_rate_for_incidents_and_deployments(): + + first_week_2024 = datetime(2024, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) + second_week_2024 = datetime(2024, 1, 8, 0, 0, 0, tzinfo=pytz.UTC) + third_week_2024 = datetime(2024, 1, 15, 0, 0, 0, tzinfo=pytz.UTC) + fourth_week_2024 = datetime(2024, 1, 22, 0, 0, 0, tzinfo=pytz.UTC) + + from_time = first_week_2024 + timedelta(days=1) + to_time = fourth_week_2024 + timedelta(days=2) + + # Week 1 + incident_0 = get_incident(creation_date=from_time + timedelta(hours=10)) + + deployment_1 = get_deployment(conducted_at=from_time + timedelta(hours=12)) + + deployment_2 = get_deployment(conducted_at=from_time + timedelta(hours=24)) + incident_1 = get_incident(creation_date=from_time + timedelta(hours=28)) + incident_2 = get_incident(creation_date=from_time + timedelta(hours=29)) + + # Week 3 + deployment_3 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=4)) + incident_3 = get_incident(creation_date=fourth_week_2024 - timedelta(days=3)) + incident_4 = get_incident(creation_date=fourth_week_2024 - timedelta(days=3)) + + deployment_4 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=2)) + + deployment_5 = get_deployment(conducted_at=fourth_week_2024 - timedelta(hours=6)) + incident_5 = get_incident(creation_date=fourth_week_2024 - timedelta(hours=3)) + incident_6 = get_incident(creation_date=fourth_week_2024 - timedelta(hours=2)) + + deployment_6 = get_deployment(conducted_at=fourth_week_2024 - timedelta(minutes=30)) + + incidents = [ + incident_0, + incident_1, + incident_2, + incident_3, + incident_4, + incident_5, + incident_6, + ] + + deployments = [ + deployment_1, + deployment_2, + deployment_3, + deployment_4, + deployment_5, + deployment_6, + ] + + incident_service = get_incident_service() + weekly_change_failure_rate = incident_service.get_weekly_change_failure_rate( + Interval(from_time, to_time), + deployments, + incidents, + ) + + assert weekly_change_failure_rate == { + first_week_2024: get_change_failure_rate_metrics( + set([deployment_2]), set([deployment_1, deployment_2]) + ), + second_week_2024: get_change_failure_rate_metrics([], []), + third_week_2024: get_change_failure_rate_metrics( + set([deployment_3, deployment_5]), + set([deployment_3, deployment_4, deployment_5, deployment_6]), + ), + fourth_week_2024: get_change_failure_rate_metrics([], []), + } diff --git a/backend/analytics_server/tests/service/Incidents/test_deployment_incident_mapper.py b/backend/analytics_server/tests/service/Incidents/test_deployment_incident_mapper.py new file mode 100644 index 000000000..c4de193c1 --- /dev/null +++ b/backend/analytics_server/tests/service/Incidents/test_deployment_incident_mapper.py @@ -0,0 +1,119 @@ +from datetime import timedelta +from dora.service.incidents.incidents import get_incident_service +from dora.utils.time import time_now + +from tests.factories.models import get_incident, get_deployment + + +# No incidents, no deployments +def test_get_deployment_incidents_count_map_returns_empty_dict_when_given_no_incidents_no_deployments(): + incident_service = get_incident_service() + incidents = [] + deployments = [] + deployment_incidents_count_map = incident_service.get_deployment_incidents_map( + deployments, + incidents, + ) + assert deployment_incidents_count_map == {} + + +# No incidents, some deployments +def test_get_deployment_incidents_count_map_returns_deployment_incident_count_map_when_given_no_incidents_some_deployments(): + incident_service = get_incident_service() + incidents = [] + deployments = [ + get_deployment(conducted_at=time_now() - timedelta(days=2)), + get_deployment(conducted_at=time_now() - timedelta(hours=6)), + ] + deployment_incidents_count_map = incident_service.get_deployment_incidents_map( + deployments, + incidents, + ) + assert deployment_incidents_count_map == {deployments[0]: [], deployments[1]: []} + + +# Some incidents, no deployments +def test_get_deployment_incidents_count_map_returns_empty_dict_when_given_some_incidents_no_deployments(): + incident_service = get_incident_service() + incidents = [get_incident(creation_date=time_now() - timedelta(days=3))] + deployments = [] + deployment_incidents_count_map = incident_service.get_deployment_incidents_map( + deployments, incidents + ) + assert deployment_incidents_count_map == {} + + +# One incident between two deployments +def test_get_deployment_incidents_count_map_returns_deployment_incident_count_map_when_given_one_incidents_bw_two_deployments(): + incident_service = get_incident_service() + incidents = [get_incident(creation_date=time_now() - timedelta(days=1))] + deployments = [ + get_deployment(conducted_at=time_now() - timedelta(days=2)), + get_deployment(conducted_at=time_now() - timedelta(hours=6)), + ] + deployment_incidents_count_map = incident_service.get_deployment_incidents_map( + deployments, incidents + ) + assert deployment_incidents_count_map == { + deployments[0]: [incidents[0]], + deployments[1]: [], + } + + +# One incident before two deployments +def test_get_deployment_incidents_count_map_returns_deployment_incident_count_map_when_given_one_incidents_bef_two_deployments(): + incident_service = get_incident_service() + incidents = [get_incident(creation_date=time_now() - timedelta(days=3))] + deployments = [ + get_deployment(conducted_at=time_now() - timedelta(days=2)), + get_deployment(conducted_at=time_now() - timedelta(hours=6)), + ] + deployment_incidents_count_map = incident_service.get_deployment_incidents_map( + deployments, incidents + ) + assert deployment_incidents_count_map == {deployments[0]: [], deployments[1]: []} + + +# One incident after two deployments +def test_get_deployment_incidents_count_map_returns_deployment_incident_count_map_when_given_one_incidents_after_two_deployments(): + incident_service = get_incident_service() + incidents = [get_incident(creation_date=time_now() - timedelta(hours=1))] + deployments = [ + get_deployment(conducted_at=time_now() - timedelta(days=2)), + get_deployment(conducted_at=time_now() - timedelta(hours=6)), + ] + deployment_incidents_count_map = incident_service.get_deployment_incidents_map( + deployments, incidents + ) + assert deployment_incidents_count_map == { + deployments[0]: [], + deployments[1]: [incidents[0]], + } + + +# Multiple incidents and deployments +def test_get_deployment_incidents_count_map_returns_deployment_incident_count_map_when_given_multi_incidents_multi_deployments(): + incident_service = get_incident_service() + incidents = [ + get_incident(creation_date=time_now() - timedelta(days=5)), + get_incident(creation_date=time_now() - timedelta(days=3)), + get_incident(creation_date=time_now() - timedelta(hours=20)), + get_incident(creation_date=time_now() - timedelta(hours=1)), + ] + deployments = [ + get_deployment(conducted_at=time_now() - timedelta(days=7)), + get_deployment(conducted_at=time_now() - timedelta(days=6)), + get_deployment(conducted_at=time_now() - timedelta(days=4)), + get_deployment(conducted_at=time_now() - timedelta(days=2)), + get_deployment(conducted_at=time_now() - timedelta(hours=6)), + ] + deployment_incidents_count_map = incident_service.get_deployment_incidents_map( + deployments, incidents + ) + assert deployment_incidents_count_map == { + deployments[0]: [], + deployments[1]: [incidents[0]], + deployments[2]: [incidents[1]], + deployments[3]: [incidents[2]], + deployments[4]: [incidents[3]], + } diff --git a/backend/analytics_server/tests/service/__init__.py b/backend/analytics_server/tests/service/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/service/code/__init__.py b/backend/analytics_server/tests/service/code/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/service/code/sync/__init__.py b/backend/analytics_server/tests/service/code/sync/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/service/code/sync/test_etl_code_analytics.py b/backend/analytics_server/tests/service/code/sync/test_etl_code_analytics.py new file mode 100644 index 000000000..740c0d90c --- /dev/null +++ b/backend/analytics_server/tests/service/code/sync/test_etl_code_analytics.py @@ -0,0 +1,505 @@ +from datetime import timedelta + +from dora.service.code.sync.etl_code_analytics import CodeETLAnalyticsService +from dora.store.models.code import PullRequestState, PullRequestEventState +from dora.utils.time import time_now +from tests.factories.models.code import ( + get_pull_request, + get_pull_request_event, + get_pull_request_commit, +) + + +def test_pr_performance_returns_first_review_tat_for_first_review(): + pr_service = CodeETLAnalyticsService() + t1 = time_now() + t2 = t1 + timedelta(hours=1) + pr = get_pull_request(created_at=t1, updated_at=t1) + pr_event = get_pull_request_event(pull_request_id=pr.id, created_at=t2) + performance = pr_service.get_pr_performance(pr, [pr_event]) + assert performance.first_review_time == 3600 + + +def test_pr_performance_returns_minus1_first_review_tat_for_no_reviews(): + pr_service = CodeETLAnalyticsService() + pr = get_pull_request() + performance = pr_service.get_pr_performance(pr, []) + assert performance.first_review_time == -1 + + +def test_pr_performance_returns_minus1_first_approved_review_tat_for_no_approved_review(): + pr_service = CodeETLAnalyticsService() + t1 = time_now() + t2 = t1 + timedelta(hours=1) + pr = get_pull_request(created_at=t1, updated_at=t1) + pr_event_1 = get_pull_request_event( + pull_request_id=pr.id, state="REJECTED", created_at=t2 + ) + performance = pr_service.get_pr_performance(pr, [pr_event_1]) + assert performance.merge_time == -1 + + +def test_pr_performance_returns_merge_time_minus1_for_merged_pr_without_review(): + pr_service = CodeETLAnalyticsService() + t1 = time_now() + t2 = time_now() + timedelta(minutes=30) + pr = get_pull_request( + state=PullRequestState.MERGED, state_changed_at=t2, created_at=t1, updated_at=t2 + ) + performance = pr_service.get_pr_performance(pr, []) + assert performance.merge_time == -1 + + +def test_pr_performance_returns_blocking_reviews(): + pr_service = CodeETLAnalyticsService() + t1 = time_now() + t2 = time_now() + timedelta(minutes=30) + pr = get_pull_request( + state=PullRequestState.MERGED, + requested_reviews=["abc", "bcd"], + state_changed_at=t2, + created_at=t1, + updated_at=t2, + ) + performance = pr_service.get_pr_performance(pr, []) + assert performance.blocking_reviews == 0 + + +def test_pr_performance_returns_rework_time(): + pr_service = CodeETLAnalyticsService() + t1 = time_now() + t2 = t1 + timedelta(hours=1) + t3 = t2 + timedelta(hours=1) + t4 = t3 + timedelta(hours=1) + pr = get_pull_request( + state=PullRequestState.MERGED, state_changed_at=t4, created_at=t1, updated_at=t1 + ) + changes_requested_1 = get_pull_request_event( + pull_request_id=pr.id, + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t2, + ) + comment2 = get_pull_request_event( + pull_request_id=pr.id, + state=PullRequestEventState.COMMENTED.value, + created_at=t3, + ) + approval = get_pull_request_event( + pull_request_id=pr.id, state=PullRequestEventState.APPROVED.value, created_at=t4 + ) + performance = pr_service.get_pr_performance( + pr, [changes_requested_1, comment2, approval] + ) + + assert performance.rework_time == (t4 - t2).total_seconds() + + +def test_pr_performance_returns_rework_time_0_for_approved_prs(): + pr_service = CodeETLAnalyticsService() + t1 = time_now() + t2 = t1 + timedelta(hours=1) + pr = get_pull_request( + state=PullRequestState.MERGED, state_changed_at=t2, created_at=t1, updated_at=t1 + ) + approval = get_pull_request_event( + pull_request_id=pr.id, state=PullRequestEventState.APPROVED.value, created_at=t2 + ) + performance = pr_service.get_pr_performance(pr, [approval]) + + assert performance.rework_time == 0 + + +def test_pr_performance_returns_rework_time_as_per_first_approved_prs(): + pr_service = CodeETLAnalyticsService() + t1 = time_now() + t2 = t1 + timedelta(hours=1) + t3 = t2 + timedelta(hours=1) + t4 = t3 + timedelta(hours=1) + pr = get_pull_request( + state=PullRequestState.MERGED, state_changed_at=t4, created_at=t1, updated_at=t1 + ) + changes_requested_1 = get_pull_request_event( + pull_request_id=pr.id, + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t2, + ) + approval = get_pull_request_event( + pull_request_id=pr.id, state=PullRequestEventState.APPROVED.value, created_at=t3 + ) + approval_2 = get_pull_request_event( + pull_request_id=pr.id, state=PullRequestEventState.APPROVED.value, created_at=t4 + ) + performance = pr_service.get_pr_performance( + pr, [changes_requested_1, approval, approval_2] + ) + + assert performance.rework_time == (t3 - t2).total_seconds() + + +def test_pr_performance_returns_rework_time_for_open_prs(): + pr_service = CodeETLAnalyticsService() + t1 = time_now() + t2 = t1 + timedelta(hours=1) + t3 = t2 + timedelta(hours=1) + t4 = t3 + timedelta(hours=1) + pr = get_pull_request(state=PullRequestState.OPEN, created_at=t1, updated_at=t1) + changes_requested_1 = get_pull_request_event( + pull_request_id=pr.id, + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t2, + ) + approval = get_pull_request_event( + pull_request_id=pr.id, state=PullRequestEventState.APPROVED.value, created_at=t3 + ) + performance = pr_service.get_pr_performance(pr, [changes_requested_1, approval]) + + assert performance.rework_time == (t3 - t2).total_seconds() + + +def test_pr_performance_returns_rework_time_minus1_for_non_approved_prs(): + pr_service = CodeETLAnalyticsService() + t1 = time_now() + t2 = t1 + timedelta(hours=1) + t3 = t2 + timedelta(hours=1) + t4 = t3 + timedelta(hours=1) + pr = get_pull_request(state=PullRequestState.OPEN, created_at=t1, updated_at=t1) + changes_requested_1 = get_pull_request_event( + pull_request_id=pr.id, + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t2, + ) + performance = pr_service.get_pr_performance(pr, [changes_requested_1]) + + assert performance.rework_time == -1 + + +def test_pr_performance_returns_rework_time_minus1_for_merged_prs_without_reviews(): + pr_service = CodeETLAnalyticsService() + t1 = time_now() + pr = get_pull_request( + state=PullRequestState.MERGED, state_changed_at=t1, created_at=t1, updated_at=t1 + ) + performance = pr_service.get_pr_performance(pr, []) + + assert performance.rework_time == -1 + + +def test_pr_performance_returns_cycle_time_for_merged_pr(): + pr_service = CodeETLAnalyticsService() + t1 = time_now() + t2 = t1 + timedelta(days=1) + pr = get_pull_request( + state=PullRequestState.MERGED, state_changed_at=t2, created_at=t1, updated_at=t2 + ) + performance = pr_service.get_pr_performance(pr, []) + + assert performance.cycle_time == 86400 + + +def test_pr_performance_returns_cycle_time_minus1_for_non_merged_pr(): + pr_service = CodeETLAnalyticsService() + pr = get_pull_request() + performance = pr_service.get_pr_performance(pr, []) + + assert performance.cycle_time == -1 + + +def test_pr_rework_cycles_returns_zero_cycles_when_pr_approved(): + pr_service = CodeETLAnalyticsService() + pr = get_pull_request(reviewers=["dhruv", "jayant"]) + t1 = time_now() + t2 = t1 + timedelta(seconds=1) + commit = get_pull_request_commit(pr_id=pr.id, created_at=t1) + reviews = [get_pull_request_event(reviewer="dhruv", created_at=t2)] + assert pr_service.get_rework_cycles(pr, reviews, [commit]) == 0 + + +def test_rework_cycles_returns_1_cycle_if_some_rework_done(): + pr = get_pull_request(reviewers=["dhruv", "jayant"]) + t1 = time_now() + t2 = t1 + timedelta(seconds=1) + t3 = t1 + timedelta(seconds=2) + review_1 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t1, + ) + commit = get_pull_request_commit(pr_id=pr.id, created_at=t2) + review_2 = get_pull_request_event( + pull_request_id=pr.id, reviewer="dhruv", created_at=t3 + ) + pr_service = CodeETLAnalyticsService() + assert pr_service.get_rework_cycles(pr, [review_1, review_2], [commit]) == 1 + + +def test_rework_cycles_returns_2_cycles_if_there_were_comments_between_commit_batch(): + pr = get_pull_request(reviewers=["dhruv", "jayant"]) + t1 = time_now() + t2 = t1 + timedelta(seconds=1) + t3 = t1 + timedelta(seconds=2) + t4 = t1 + timedelta(seconds=3) + t5 = t1 + timedelta(seconds=4) + review_1 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t1, + ) + commit_1 = get_pull_request_commit(pr_id=pr.id, created_at=t2) + review_2 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t3, + ) + commit_2 = get_pull_request_commit(pr_id=pr.id, created_at=t4) + review_3 = get_pull_request_event( + pull_request_id=pr.id, reviewer="dhruv", created_at=t5 + ) + pr_service = CodeETLAnalyticsService() + assert ( + pr_service.get_rework_cycles( + pr, [review_1, review_2, review_3], [commit_1, commit_2] + ) + == 2 + ) + + +def test_rework_cycles_returns_1_cycle_despite_multiple_commits(): + pr = get_pull_request(reviewers=["dhruv", "jayant"]) + t1 = time_now() + t2 = t1 + timedelta(seconds=1) + t3 = t1 + timedelta(seconds=2) + review_1 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t1, + ) + commit_1 = get_pull_request_commit(pr_id=pr.id, created_at=t2) + commit_2 = get_pull_request_commit(pr_id=pr.id, created_at=t2) + commit_3 = get_pull_request_commit(pr_id=pr.id, created_at=t2) + review_2 = get_pull_request_event( + pull_request_id=pr.id, reviewer="dhruv", created_at=t3 + ) + pr_service = CodeETLAnalyticsService() + assert ( + pr_service.get_rework_cycles( + pr, [review_1, review_2], [commit_1, commit_2, commit_3] + ) + == 1 + ) + + +def test_rework_cycles_returns_2_cycles_despite_multiple_comments(): + pr = get_pull_request(reviewers=["dhruv", "jayant"]) + t1 = time_now() + t2 = t1 + timedelta(seconds=1) + t3 = t1 + timedelta(seconds=2) + t4 = t1 + timedelta(seconds=3) + t5 = t1 + timedelta(seconds=4) + review_1 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.COMMENTED.value, + created_at=t1, + ) + review_2 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.COMMENTED.value, + created_at=t1, + ) + review_3 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t1, + ) + commit_1 = get_pull_request_commit(pr_id=pr.id, created_at=t2) + review_4 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t3, + ) + commit_2 = get_pull_request_commit(pr_id=pr.id, created_at=t4) + review_5 = get_pull_request_event( + pull_request_id=pr.id, reviewer="dhruv", created_at=t5 + ) + review_6 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.COMMENTED.value, + created_at=t5, + ) + pr_service = CodeETLAnalyticsService() + assert ( + pr_service.get_rework_cycles( + pr, + [review_1, review_2, review_3, review_4, review_5, review_6], + [commit_1, commit_2], + ) + == 2 + ) + + +def test_rework_cycles_doesnt_count_commits_post_first_approval(): + pr = get_pull_request(reviewers=["dhruv", "jayant"]) + t1 = time_now() + t2 = t1 + timedelta(seconds=1) + t3 = t1 + timedelta(seconds=2) + t4 = t1 + timedelta(seconds=3) + t5 = t1 + timedelta(seconds=4) + t6 = t1 + timedelta(seconds=5) + review_1 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t1, + ) + commit_1 = get_pull_request_commit(pr_id=pr.id, created_at=t2) + review_2 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.COMMENTED.value, + created_at=t3, + ) + commit_2 = get_pull_request_commit(pr_id=pr.id, created_at=t4) + review_3 = get_pull_request_event( + pull_request_id=pr.id, reviewer="dhruv", created_at=t5 + ) + commit_3 = get_pull_request_commit(pr_id=pr.id, created_at=t6) + commit_4 = get_pull_request_commit(pr_id=pr.id, created_at=t6) + pr_service = CodeETLAnalyticsService() + assert ( + pr_service.get_rework_cycles( + pr, [review_1, review_2, review_3], [commit_1, commit_2, commit_3, commit_4] + ) + == 2 + ) + + +def test_rework_cycles_returns_0_for_unapproved_pr(): + pr = get_pull_request(reviewers=["dhruv", "jayant"]) + t1 = time_now() + t2 = t1 + timedelta(seconds=1) + t3 = t1 + timedelta(seconds=2) + t4 = t1 + timedelta(seconds=3) + t5 = t1 + timedelta(seconds=4) + t6 = t1 + timedelta(seconds=5) + review_1 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t1, + ) + commit_1 = get_pull_request_commit(pr_id=pr.id, created_at=t2) + review_2 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t3, + ) + commit_2 = get_pull_request_commit(pr_id=pr.id, created_at=t4) + review_3 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t5, + ) + commit_3 = get_pull_request_commit(pr_id=pr.id, created_at=t6) + commit_4 = get_pull_request_commit(pr_id=pr.id, created_at=t6) + pr_service = CodeETLAnalyticsService() + assert ( + pr_service.get_rework_cycles( + pr, [review_1, review_2, review_3], [commit_1, commit_2, commit_3, commit_4] + ) + == 0 + ) + + +def test_rework_cycles_returs_0_for_non_reviewer_comments(): + pr = get_pull_request(reviewers=["dhruv", "jayant"]) + t1 = time_now() + t2 = t1 + timedelta(seconds=1) + t3 = t1 + timedelta(seconds=2) + t4 = t1 + timedelta(seconds=3) + t5 = t1 + timedelta(seconds=4) + t6 = t1 + timedelta(seconds=5) + review_1 = get_pull_request_event( + pull_request_id=pr.id, + state=PullRequestEventState.COMMENTED.value, + created_at=t1, + ) + commit_1 = get_pull_request_commit(pr_id=pr.id, created_at=t2) + review_2 = get_pull_request_event( + pull_request_id=pr.id, + state=PullRequestEventState.COMMENTED.value, + created_at=t3, + ) + commit_2 = get_pull_request_commit(pr_id=pr.id, created_at=t4) + review_3 = get_pull_request_event( + pull_request_id=pr.id, + state=PullRequestEventState.COMMENTED.value, + created_at=t5, + ) + commit_3 = get_pull_request_commit(pr_id=pr.id, created_at=t6) + review_4 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.APPROVED.value, + created_at=t5, + ) + pr_service = CodeETLAnalyticsService() + assert ( + pr_service.get_rework_cycles( + pr, [review_1, review_2, review_3, review_4], [commit_1, commit_2, commit_3] + ) + == 0 + ) + + +def test_rework_cycles_returs_1_for_multiple_approvals(): + pr = get_pull_request(reviewers=["dhruv", "jayant"]) + t1 = time_now() + t2 = t1 + timedelta(seconds=1) + t3 = t1 + timedelta(seconds=2) + t4 = t1 + timedelta(seconds=3) + t5 = t1 + timedelta(seconds=4) + t6 = t1 + timedelta(seconds=5) + t7 = t1 + timedelta(seconds=6) + review_1 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.CHANGES_REQUESTED.value, + created_at=t1, + ) + commit_1 = get_pull_request_commit(pr_id=pr.id, created_at=t2) + review_2 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.APPROVED.value, + created_at=t3, + ) + commit_2 = get_pull_request_commit(pr_id=pr.id, created_at=t4) + review_3 = get_pull_request_event( + pull_request_id=pr.id, + state=PullRequestEventState.COMMENTED.value, + created_at=t5, + ) + commit_3 = get_pull_request_commit(pr_id=pr.id, created_at=t6) + review_4 = get_pull_request_event( + pull_request_id=pr.id, + reviewer="dhruv", + state=PullRequestEventState.APPROVED.value, + created_at=t7, + ) + pr_service = CodeETLAnalyticsService() + assert ( + pr_service.get_rework_cycles( + pr, [review_1, review_2, review_3, review_4], [commit_1, commit_2, commit_3] + ) + == 1 + ) diff --git a/backend/analytics_server/tests/service/code/sync/test_etl_github_handler.py b/backend/analytics_server/tests/service/code/sync/test_etl_github_handler.py new file mode 100644 index 000000000..287706717 --- /dev/null +++ b/backend/analytics_server/tests/service/code/sync/test_etl_github_handler.py @@ -0,0 +1,353 @@ +from datetime import datetime + +import pytz + +from dora.service.code.sync.etl_github_handler import GithubETLHandler +from dora.store.models.code import PullRequestState +from dora.utils.string import uuid4_str +from tests.factories.models import ( + get_pull_request, + get_pull_request_commit, + get_pull_request_event, +) +from tests.factories.models.exapi.github import ( + get_github_commit_dict, + get_github_pull_request_review, + get_github_pull_request, +) +from tests.utilities import compare_objects_as_dicts + +ORG_ID = uuid4_str() + + +def test__to_pr_model_given_a_github_pr_returns_new_pr_model(): + repo_id = uuid4_str() + number = 123 + user_login = "abc" + merged_at = datetime(2022, 6, 29, 10, 53, 15, tzinfo=pytz.UTC) + head_branch = "feature" + base_branch = "main" + title = "random_title" + review_comments = 3 + merge_commit_sha = "123456789098765" + + github_pull_request = get_github_pull_request( + number=number, + merged_at=merged_at, + head_ref=head_branch, + base_ref=base_branch, + user_login=user_login, + merge_commit_sha=merge_commit_sha, + commits=3, + additions=10, + deletions=5, + changed_files=2, + ) + + github_etl_handler = GithubETLHandler(ORG_ID, None, None, None, None) + pr_model = github_etl_handler._to_pr_model( + pr=github_pull_request, + pr_model=None, + repo_id=repo_id, + review_comments=review_comments, + ) + + expected_pr_model = get_pull_request( + repo_id=repo_id, + number=str(number), + author=str(user_login), + state=PullRequestState.MERGED, + title=title, + head_branch=head_branch, + base_branch=base_branch, + provider="github", + requested_reviews=[], + data=github_pull_request.raw_data, + state_changed_at=merged_at, + meta={ + "code_stats": { + "commits": 3, + "additions": 10, + "deletions": 5, + "changed_files": 2, + "comments": review_comments, + }, + "user_profile": { + "username": user_login, + }, + }, + reviewers=[], + merge_commit_sha=merge_commit_sha, + ) + # Ignoring the following fields as they are generated as side effects and are not part of the actual data + # reviewers, rework_time, first_commit_to_open, first_response_time, lead_time, merge_time, merge_to_deploy, cycle_time + assert ( + compare_objects_as_dicts( + pr_model, + expected_pr_model, + [ + "id", + "created_at", + "updated_at", + "reviewers", + "rework_time", + "first_commit_to_open", + "first_response_time", + "lead_time", + "merge_time", + "merge_to_deploy", + "cycle_time", + ], + ) + is True + ) + + +def test__to_pr_model_given_a_github_pr_and_db_pr_returns_updated_pr_model(): + repo_id = uuid4_str() + number = 123 + user_login = "abc" + merged_at = datetime(2022, 6, 29, 10, 53, 15, tzinfo=pytz.UTC) + head_branch = "feature" + base_branch = "main" + title = "random_title" + review_comments = 3 + merge_commit_sha = "123456789098765" + + github_pull_request = get_github_pull_request( + number=number, + merged_at=merged_at, + head_ref=head_branch, + base_ref=base_branch, + user_login=user_login, + merge_commit_sha=merge_commit_sha, + commits=3, + additions=10, + deletions=5, + changed_files=2, + ) + + given_pr_model = get_pull_request( + repo_id=repo_id, + number=str(number), + provider="github", + ) + + github_etl_handler = GithubETLHandler(ORG_ID, None, None, None, None) + pr_model = github_etl_handler._to_pr_model( + pr=github_pull_request, + pr_model=given_pr_model, + repo_id=repo_id, + review_comments=review_comments, + ) + + expected_pr_model = get_pull_request( + id=given_pr_model.id, + repo_id=repo_id, + number=str(number), + author=str(user_login), + state=PullRequestState.MERGED, + title=title, + head_branch=head_branch, + base_branch=base_branch, + provider="github", + requested_reviews=[], + data=github_pull_request.raw_data, + state_changed_at=merged_at, + meta={ + "code_stats": { + "commits": 3, + "additions": 10, + "deletions": 5, + "changed_files": 2, + "comments": review_comments, + }, + "user_profile": { + "username": user_login, + }, + }, + reviewers=[], + merge_commit_sha=merge_commit_sha, + ) + # Ignoring the following fields as they are generated as side effects and are not part of the actual data + # reviewers, rework_time, first_commit_to_open, first_response_time, lead_time, merge_time, merge_to_deploy, cycle_time + assert ( + compare_objects_as_dicts( + pr_model, + expected_pr_model, + [ + "created_at", + "updated_at", + "reviewers", + "rework_time", + "first_commit_to_open", + "first_response_time", + "lead_time", + "merge_time", + "merge_to_deploy", + "cycle_time", + ], + ) + is True + ) + + +def test__to_pr_events_given_an_empty_list_of_events_returns_an_empty_list(): + pr_model = get_pull_request() + assert GithubETLHandler._to_pr_events([], pr_model, []) == [] + + +def test__to_pr_events_given_a_list_of_only_new_events_returns_a_list_of_pr_events(): + pr_model = get_pull_request() + event1 = get_github_pull_request_review() + event2 = get_github_pull_request_review() + events = [event1, event2] + + pr_events = GithubETLHandler._to_pr_events(events, pr_model, []) + + expected_pr_events = [ + get_pull_request_event( + pull_request_id=str(pr_model.id), + org_repo_id=pr_model.repo_id, + data=event1.raw_data, + created_at=event1.submitted_at, + type="REVIEW", + idempotency_key=event1.id, + reviewer=event1.user_login, + ), + get_pull_request_event( + pull_request_id=str(pr_model.id), + org_repo_id=pr_model.repo_id, + data=event2.raw_data, + created_at=event2.submitted_at, + type="REVIEW", + idempotency_key=event2.id, + reviewer=event2.user_login, + ), + ] + + for event, expected_event in zip(pr_events, expected_pr_events): + assert compare_objects_as_dicts(event, expected_event, ["id"]) is True + + +def test__to_pr_events_given_a_list_of_new_events_and_old_events_returns_a_list_of_pr_events(): + pr_model = get_pull_request() + event1 = get_github_pull_request_review() + event2 = get_github_pull_request_review() + events = [event1, event2] + + old_event = get_pull_request_event( + pull_request_id=str(pr_model.id), + org_repo_id=pr_model.repo_id, + data=event1.raw_data, + created_at=event1.submitted_at, + type="REVIEW", + idempotency_key=event1.id, + reviewer=event1.user_login, + ) + + pr_events = GithubETLHandler._to_pr_events(events, pr_model, [old_event]) + + expected_pr_events = [ + old_event, + get_pull_request_event( + pull_request_id=str(pr_model.id), + org_repo_id=pr_model.repo_id, + data=event2.raw_data, + created_at=event2.submitted_at, + type="REVIEW", + idempotency_key=event2.id, + reviewer=event2.user_login, + ), + ] + + for event, expected_event in zip(pr_events, expected_pr_events): + assert compare_objects_as_dicts(event, expected_event, ["id"]) is True + + +def test__to_pr_commits_given_an_empty_list_of_commits_returns_an_empty_list(): + pr_model = get_pull_request() + github_etl_handler = GithubETLHandler(ORG_ID, None, None, None, None) + assert github_etl_handler._to_pr_commits([], pr_model) == [] + + +def test__to_pr_commits_given_a_list_of_commits_returns_a_list_of_pr_commits(): + pr_model = get_pull_request() + common_url = "random_url" + common_message = "random_message" + sha1 = "123456789098765" + author1 = "author_abc" + created_at1 = "2022-06-29T10:53:15Z" + commit1 = get_github_commit_dict( + sha=sha1, + author_login=author1, + created_at=created_at1, + url=common_url, + message=common_message, + ) + sha2 = "987654321234567" + author2 = "author_xyz" + created_at2 = "2022-06-29T12:53:15Z" + commit2 = get_github_commit_dict( + sha=sha2, + author_login=author2, + created_at=created_at2, + url=common_url, + message=common_message, + ) + sha3 = "543216789098765" + author3 = "author_abc" + created_at3 = "2022-06-29T15:53:15Z" + commit3 = get_github_commit_dict( + sha=sha3, + author_login=author3, + created_at=created_at3, + url=common_url, + message=common_message, + ) + + commits = [commit1, commit2, commit3] + github_etl_handler = GithubETLHandler(ORG_ID, None, None, None, None) + pr_commits = github_etl_handler._to_pr_commits(commits, pr_model) + + expected_pr_commits = [ + get_pull_request_commit( + pr_id=str(pr_model.id), + org_repo_id=pr_model.repo_id, + hash=sha1, + author=author1, + url=common_url, + message=common_message, + created_at=datetime(2022, 6, 29, 10, 53, 15, tzinfo=pytz.UTC), + data=commit1, + ), + get_pull_request_commit( + pr_id=str(pr_model.id), + org_repo_id=pr_model.repo_id, + hash=sha2, + author=author2, + url=common_url, + message=common_message, + created_at=datetime(2022, 6, 29, 12, 53, 15, tzinfo=pytz.UTC), + data=commit2, + ), + get_pull_request_commit( + pr_id=str(pr_model.id), + org_repo_id=pr_model.repo_id, + hash=sha3, + author=author3, + url=common_url, + message=common_message, + created_at=datetime(2022, 6, 29, 15, 53, 15, tzinfo=pytz.UTC), + data=commit3, + ), + ] + + for commit, expected_commit in zip(pr_commits, expected_pr_commits): + assert compare_objects_as_dicts(commit, expected_commit) is True + + +def test__dt_from_github_dt_string_given_date_string_returns_correct_datetime(): + date_string = "2024-04-18T10:53:15Z" + expected = datetime(2024, 4, 18, 10, 53, 15, tzinfo=pytz.UTC) + assert GithubETLHandler._dt_from_github_dt_string(date_string) == expected diff --git a/backend/analytics_server/tests/service/code/test_lead_time_service.py b/backend/analytics_server/tests/service/code/test_lead_time_service.py new file mode 100644 index 000000000..48b589e8b --- /dev/null +++ b/backend/analytics_server/tests/service/code/test_lead_time_service.py @@ -0,0 +1,198 @@ +from dora.utils.string import uuid4_str +from tests.utilities import compare_objects_as_dicts +from dora.service.code.lead_time import LeadTimeService +from dora.service.code.models.lead_time import LeadTimeMetrics + + +class FakeCodeRepoService: + pass + + +class FakeDeploymentsService: + pass + + +def test_get_avg_time_for_multiple_lead_time_metrics_returns_correct_average(): + lead_time_metrics = [ + LeadTimeMetrics(first_response_time=1, pr_count=1), + LeadTimeMetrics(first_response_time=2, pr_count=1), + ] + field = "first_response_time" + + lead_time_service = LeadTimeService(FakeCodeRepoService, FakeDeploymentsService) + + result = lead_time_service._get_avg_time(lead_time_metrics, field) + assert result == 1.5 + + +def test_get_avg_time_for_different_lead_time_metrics_given_returns_correct_average(): + lead_time_metrics = [ + LeadTimeMetrics(first_response_time=1, pr_count=1), + LeadTimeMetrics(first_response_time=0, pr_count=0), + LeadTimeMetrics(first_response_time=3, pr_count=1), + ] + field = "first_response_time" + + lead_time_service = LeadTimeService(FakeCodeRepoService, FakeDeploymentsService) + + result = lead_time_service._get_avg_time(lead_time_metrics, field) + assert result == 2 + + +def test_get_avg_time_for_no_lead_time_metrics_returns_zero(): + lead_time_metrics = [] + field = "first_response_time" + + lead_time_service = LeadTimeService(FakeCodeRepoService, FakeDeploymentsService) + + result = lead_time_service._get_avg_time(lead_time_metrics, field) + assert result == 0 + + +def test_get_avg_time_for_empty_lead_time_metrics_returns_zero(): + lead_time_metrics = [LeadTimeMetrics(), LeadTimeMetrics()] + field = "first_response_time" + + lead_time_service = LeadTimeService(FakeCodeRepoService, FakeDeploymentsService) + + result = lead_time_service._get_avg_time(lead_time_metrics, field) + assert result == 0 + + +def test_get_weighted_avg_lead_time_metrics_returns_correct_average(): + lead_time_metrics = [ + LeadTimeMetrics( + first_commit_to_open=1, + first_response_time=4, + rework_time=10, + merge_time=1, + merge_to_deploy=1, + pr_count=1, + ), + LeadTimeMetrics( + first_commit_to_open=2, + first_response_time=3, + rework_time=10, + merge_time=1, + merge_to_deploy=1, + pr_count=2, + ), + LeadTimeMetrics( + first_commit_to_open=3, + first_response_time=2, + rework_time=10, + merge_time=1, + merge_to_deploy=1, + pr_count=3, + ), + LeadTimeMetrics( + first_commit_to_open=4, + first_response_time=1, + rework_time=10, + merge_time=1, + merge_to_deploy=1, + pr_count=4, + ), + ] + + lead_time_service = LeadTimeService(FakeCodeRepoService, FakeDeploymentsService) + + result = lead_time_service._get_weighted_avg_lead_time_metrics(lead_time_metrics) + + expected = LeadTimeMetrics( + first_commit_to_open=3, + first_response_time=2, + rework_time=10, + merge_time=1, + merge_to_deploy=1, + pr_count=10, + ) + assert compare_objects_as_dicts(result, expected) + + +def test_get_teams_avg_lead_time_metrics_returns_correct_values(): + + team_1 = uuid4_str() + team_2 = uuid4_str() + + team_lead_time_metrics = { + team_1: [ + LeadTimeMetrics( + first_commit_to_open=1, + first_response_time=4, + rework_time=10, + merge_time=1, + merge_to_deploy=1, + pr_count=1, + ), + LeadTimeMetrics( + first_commit_to_open=2, + first_response_time=3, + rework_time=10, + merge_time=1, + merge_to_deploy=1, + pr_count=2, + ), + LeadTimeMetrics( + first_commit_to_open=3, + first_response_time=2, + rework_time=10, + merge_time=1, + merge_to_deploy=1, + pr_count=3, + ), + LeadTimeMetrics( + first_commit_to_open=4, + first_response_time=1, + rework_time=10, + merge_time=1, + merge_to_deploy=1, + pr_count=4, + ), + ], + team_2: [ + LeadTimeMetrics( + first_commit_to_open=1, + first_response_time=4, + rework_time=10, + merge_time=1, + merge_to_deploy=1, + pr_count=1, + ), + LeadTimeMetrics( + first_commit_to_open=2, + first_response_time=3, + rework_time=10, + merge_time=1, + merge_to_deploy=1, + pr_count=2, + ), + ], + } + + lead_time_service = LeadTimeService(FakeCodeRepoService, FakeDeploymentsService) + + result = lead_time_service.get_avg_lead_time_metrics_from_map( + team_lead_time_metrics + ) + + expected = { + team_1: LeadTimeMetrics( + first_commit_to_open=3, + first_response_time=2, + rework_time=10, + merge_time=1, + merge_to_deploy=1, + pr_count=10, + ), + team_2: LeadTimeMetrics( + first_commit_to_open=5 / 3, + first_response_time=10 / 3, + rework_time=10, + merge_time=1, + merge_to_deploy=1, + pr_count=3, + ), + } + assert compare_objects_as_dicts(result[team_1], expected[team_1]) + assert compare_objects_as_dicts(result[team_2], expected[team_2]) diff --git a/backend/analytics_server/tests/service/deployments/__init__.py b/backend/analytics_server/tests/service/deployments/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/service/deployments/test_deployment_frequency.py b/backend/analytics_server/tests/service/deployments/test_deployment_frequency.py new file mode 100644 index 000000000..c84df8680 --- /dev/null +++ b/backend/analytics_server/tests/service/deployments/test_deployment_frequency.py @@ -0,0 +1,181 @@ +from datetime import datetime, timedelta + +import pytz +from dora.service.deployments.analytics import DeploymentAnalyticsService +from dora.utils.time import Interval +from tests.factories.models.code import get_deployment, get_deployment_frequency_metrics + +first_week_2024 = datetime(2024, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) +second_week_2024 = datetime(2024, 1, 8, 0, 0, 0, tzinfo=pytz.UTC) +third_week_2024 = datetime(2024, 1, 15, 0, 0, 0, tzinfo=pytz.UTC) +fourth_week_2024 = datetime(2024, 1, 22, 0, 0, 0, tzinfo=pytz.UTC) + + +def test_deployment_frequency_for_no_deployments(): + + from_time = first_week_2024 + timedelta(days=1) + to_time = third_week_2024 + timedelta(days=2) + + deployment_analytics_service = DeploymentAnalyticsService(None, None) + + assert ( + deployment_analytics_service._get_deployment_frequency_metrics( + [], Interval(from_time, to_time) + ) + == get_deployment_frequency_metrics() + ) + + +def test_deployment_frequency_for_deployments_across_days(): + + from_time = first_week_2024 + timedelta(days=1) + to_time = first_week_2024 + timedelta(days=4) + + deployment_1 = get_deployment(conducted_at=from_time + timedelta(hours=12)) + deployment_2 = get_deployment(conducted_at=from_time + timedelta(days=1)) + deployment_3 = get_deployment(conducted_at=from_time + timedelta(days=2)) + + deployment_outside_interval = get_deployment( + conducted_at=to_time + timedelta(days=20) + ) + + deployment_analytics_service = DeploymentAnalyticsService(None, None) + + assert deployment_analytics_service._get_deployment_frequency_metrics( + [deployment_1, deployment_2, deployment_3, deployment_outside_interval], + Interval(from_time, to_time), + ) == get_deployment_frequency_metrics(3, 0, 3, 3) + + +def test_deployment_frequency_for_deployments_across_weeks(): + + from_time = first_week_2024 + timedelta(days=1) + to_time = fourth_week_2024 + timedelta(days=1) + + # Week 1 + + deployment_1 = get_deployment(conducted_at=from_time + timedelta(hours=12)) + deployment_2 = get_deployment(conducted_at=from_time + timedelta(hours=24)) + + # Week 3 + deployment_3 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=4)) + deployment_4 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=2)) + deployment_5 = get_deployment(conducted_at=fourth_week_2024 - timedelta(hours=6)) + deployment_6 = get_deployment(conducted_at=fourth_week_2024 - timedelta(minutes=30)) + + deployment_analytics_service = DeploymentAnalyticsService(None, None) + + assert deployment_analytics_service._get_deployment_frequency_metrics( + [ + deployment_1, + deployment_2, + deployment_3, + deployment_4, + deployment_5, + deployment_6, + ], + Interval(from_time, to_time), + ) == get_deployment_frequency_metrics(6, 0, 1, 6) + + +def test_deployment_frequency_for_deployments_across_months(): + + from_time = first_week_2024 + timedelta(days=1) + to_time = datetime(2024, 3, 31, 0, 0, 0, tzinfo=pytz.UTC) + + second_month_2024 = datetime(2024, 2, 1, 0, 0, 0, tzinfo=pytz.UTC) + + print((to_time - from_time).days) + + # Month 1 + + deployment_1 = get_deployment(conducted_at=from_time + timedelta(hours=12)) + deployment_2 = get_deployment(conducted_at=from_time + timedelta(hours=24)) + deployment_3 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=4)) + deployment_4 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=2)) + deployment_5 = get_deployment(conducted_at=fourth_week_2024 - timedelta(hours=6)) + deployment_6 = get_deployment(conducted_at=fourth_week_2024 - timedelta(minutes=30)) + + # Month 2 + + deployment_7 = get_deployment(conducted_at=second_month_2024 + timedelta(days=3)) + deployment_8 = get_deployment(conducted_at=second_month_2024 + timedelta(days=2)) + deployment_9 = get_deployment( + conducted_at=second_month_2024 + timedelta(minutes=30) + ) + + # Month 3 + + deployment_10 = get_deployment(conducted_at=to_time - timedelta(days=3)) + deployment_11 = get_deployment(conducted_at=to_time - timedelta(days=2)) + deployment_12 = get_deployment(conducted_at=to_time - timedelta(days=1)) + deployment_13 = get_deployment(conducted_at=to_time - timedelta(days=1)) + + deployment_analytics_service = DeploymentAnalyticsService(None, None) + + assert deployment_analytics_service._get_deployment_frequency_metrics( + [ + deployment_1, + deployment_2, + deployment_3, + deployment_4, + deployment_5, + deployment_6, + deployment_7, + deployment_8, + deployment_9, + deployment_10, + deployment_11, + deployment_12, + deployment_13, + ], + Interval(from_time, to_time), + ) == get_deployment_frequency_metrics(13, 0, 1, 4) + + +def test_weekly_deployment_frequency_trends_for_no_deployments(): + + from_time = first_week_2024 + timedelta(days=1) + to_time = third_week_2024 + timedelta(days=2) + + deployment_analytics_service = DeploymentAnalyticsService(None, None) + + assert deployment_analytics_service._get_weekly_deployment_frequency_trends( + [], Interval(from_time, to_time) + ) == {first_week_2024: 0, second_week_2024: 0, third_week_2024: 0} + + +def test_weekly_deployment_frequency_trends_for_deployments(): + + from_time = first_week_2024 + timedelta(days=1) + to_time = fourth_week_2024 + timedelta(days=1) + + # Week 1 + + deployment_1 = get_deployment(conducted_at=from_time + timedelta(hours=12)) + deployment_2 = get_deployment(conducted_at=from_time + timedelta(hours=24)) + + # Week 3 + deployment_3 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=4)) + deployment_4 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=2)) + deployment_5 = get_deployment(conducted_at=fourth_week_2024 - timedelta(hours=6)) + deployment_6 = get_deployment(conducted_at=fourth_week_2024 - timedelta(minutes=30)) + + deployment_analytics_service = DeploymentAnalyticsService(None, None) + + assert deployment_analytics_service._get_weekly_deployment_frequency_trends( + [ + deployment_1, + deployment_2, + deployment_3, + deployment_4, + deployment_5, + deployment_6, + ], + Interval(from_time, to_time), + ) == { + first_week_2024: 2, + second_week_2024: 0, + third_week_2024: 4, + fourth_week_2024: 0, + } diff --git a/backend/analytics_server/tests/service/deployments/test_deployment_pr_mapper.py b/backend/analytics_server/tests/service/deployments/test_deployment_pr_mapper.py new file mode 100644 index 000000000..cdbf8f88d --- /dev/null +++ b/backend/analytics_server/tests/service/deployments/test_deployment_pr_mapper.py @@ -0,0 +1,183 @@ +from datetime import timedelta + +from dora.service.deployments.deployment_pr_mapper import DeploymentPRMapperService +from dora.store.models.code import PullRequestState +from dora.utils.time import time_now +from tests.factories.models.code import get_pull_request, get_repo_workflow_run + + +def test_deployment_pr_mapper_picks_prs_directly_merged_to_head_branch(): + t = time_now() + dep_branch = "release" + pr_to_main = get_pull_request( + state=PullRequestState.MERGED, + head_branch="feature", + base_branch="main", + state_changed_at=t, + ) + pr_to_release = get_pull_request( + state=PullRequestState.MERGED, + head_branch="feature", + base_branch="release", + state_changed_at=t + timedelta(days=2), + ) + assert DeploymentPRMapperService().get_all_prs_deployed( + [pr_to_main, pr_to_release], + get_repo_workflow_run( + head_branch=dep_branch, conducted_at=t + timedelta(days=7) + ), + ) == [pr_to_release] + + +def test_deployment_pr_mapper_ignores_prs_not_related_to_head_branch_directly_or_indirectly(): + t = time_now() + dep_branch = "release2" + pr_to_main = get_pull_request( + state=PullRequestState.MERGED, + head_branch="feature", + base_branch="main", + state_changed_at=t + timedelta(days=1), + ) + pr_to_release = get_pull_request( + state=PullRequestState.MERGED, + head_branch="feature", + base_branch="release", + state_changed_at=t + timedelta(days=2), + ) + assert ( + DeploymentPRMapperService().get_all_prs_deployed( + [pr_to_main, pr_to_release], + get_repo_workflow_run( + head_branch=dep_branch, conducted_at=t + timedelta(days=7) + ), + ) + == [] + ) + + +def test_deployment_pr_mapper_picks_prs_on_the_path_to_head_branch(): + t = time_now() + dep_branch = "release" + pr_to_feature = get_pull_request( + state=PullRequestState.MERGED, + head_branch="custom_feature", + base_branch="feature", + ) + pr_to_main = get_pull_request( + state=PullRequestState.MERGED, + head_branch="feature", + base_branch="main", + state_changed_at=t + timedelta(days=2), + ) + pr_to_release = get_pull_request( + state=PullRequestState.MERGED, + head_branch="main", + base_branch="release", + state_changed_at=t + timedelta(days=4), + ) + assert sorted( + [ + x.id + for x in DeploymentPRMapperService().get_all_prs_deployed( + [pr_to_feature, pr_to_main, pr_to_release], + get_repo_workflow_run( + head_branch=dep_branch, conducted_at=t + timedelta(days=7) + ), + ) + ] + ) == sorted([x.id for x in [pr_to_main, pr_to_release, pr_to_feature]]) + + +def test_deployment_pr_mapper_doesnt_pick_any_pr_if_no_pr_merged_to_head_branch(): + t = time_now() + dep_branch = "release" + pr_to_main = get_pull_request( + state=PullRequestState.MERGED, + head_branch="feature", + base_branch="main", + state_changed_at=t, + ) + assert ( + DeploymentPRMapperService().get_all_prs_deployed( + [pr_to_main], + get_repo_workflow_run( + head_branch=dep_branch, conducted_at=t + timedelta(days=4) + ), + ) + == [] + ) + + +def test_deployment_pr_mapper_picks_only_merged_prs_not_open_or_closed(): + t = time_now() + dep_branch = "release" + pr_to_feature = get_pull_request( + state=PullRequestState.OPEN, + head_branch="custom_feature", + base_branch="feature", + created_at=t, + state_changed_at=None, + ) + pr_to_main = get_pull_request( + state=PullRequestState.MERGED, + head_branch="feature", + base_branch="main", + state_changed_at=t + timedelta(days=2), + ) + pr_to_release = get_pull_request( + state=PullRequestState.CLOSED, + head_branch="main", + base_branch="release", + state_changed_at=t + timedelta(days=3), + ) + assert ( + DeploymentPRMapperService().get_all_prs_deployed( + [pr_to_feature, pr_to_main, pr_to_release], + get_repo_workflow_run( + head_branch=dep_branch, conducted_at=t + timedelta(days=7) + ), + ) + == [] + ) + + +def test_deployment_pr_mapper_returns_empty_for_no_prs(): + dep_branch = "release" + assert ( + DeploymentPRMapperService().get_all_prs_deployed( + [], get_repo_workflow_run(head_branch=dep_branch) + ) + == [] + ) + + +def test_deployment_pr_mapper_ignores_sub_prs_merged_post_main_pr_merge(): + dep_branch = "release" + t = time_now() + first_feature_pr = get_pull_request( + state=PullRequestState.MERGED, + head_branch="feature", + base_branch="master", + state_changed_at=t, + ) + second_feature_pr = get_pull_request( + state=PullRequestState.MERGED, + head_branch="feature", + base_branch="master", + state_changed_at=t + timedelta(days=4), + ) + pr_to_release = get_pull_request( + state=PullRequestState.MERGED, + head_branch="master", + base_branch="release", + state_changed_at=t + timedelta(days=2), + ) + + prs = DeploymentPRMapperService().get_all_prs_deployed( + [first_feature_pr, second_feature_pr, pr_to_release], + get_repo_workflow_run( + head_branch=dep_branch, conducted_at=t + timedelta(days=5) + ), + ) + + assert sorted(prs) == sorted([first_feature_pr, pr_to_release]) diff --git a/backend/analytics_server/tests/service/workflows/__init__.py b/backend/analytics_server/tests/service/workflows/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/service/workflows/sync/__init__.py b/backend/analytics_server/tests/service/workflows/sync/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/service/workflows/sync/test_etl_github_actions_handler.py b/backend/analytics_server/tests/service/workflows/sync/test_etl_github_actions_handler.py new file mode 100644 index 000000000..19c457ed9 --- /dev/null +++ b/backend/analytics_server/tests/service/workflows/sync/test_etl_github_actions_handler.py @@ -0,0 +1,110 @@ +from dora.service.workflows.sync.etl_github_actions_handler import ( + GithubActionsETLHandler, +) +from dora.store.models.code import RepoWorkflowRunsStatus +from dora.utils.string import uuid4_str +from tests.factories.models import get_repo_workflow_run +from tests.factories.models.exapi.github import get_github_workflow_run_dict +from tests.utilities import compare_objects_as_dicts + + +def test__adapt_github_workflows_to_workflow_runs_given_new_workflow_run_return_new_run(): + class WorkflowRepoService: + def get_repo_workflow_run_by_provider_workflow_run_id(self, *args): + return None + + github_workflow_run = get_github_workflow_run_dict() + org_id = uuid4_str() + repo_id = uuid4_str() + gh_actions_etl_handler = GithubActionsETLHandler(org_id, None, WorkflowRepoService) + actual_workflow_run = ( + gh_actions_etl_handler._adapt_github_workflows_to_workflow_runs( + repo_id, github_workflow_run + ) + ) + + expected_workflow_run = get_repo_workflow_run( + repo_workflow_id=repo_id, + provider_workflow_run_id=str(github_workflow_run["id"]), + event_actor=github_workflow_run["actor"]["login"], + head_branch=github_workflow_run["head_branch"], + status=RepoWorkflowRunsStatus.SUCCESS, + conducted_at=gh_actions_etl_handler._get_datetime_from_gh_datetime( + github_workflow_run["run_started_at"] + ), + duration=gh_actions_etl_handler._get_repo_workflow_run_duration( + github_workflow_run + ), + meta=github_workflow_run, + html_url=github_workflow_run["html_url"], + ) + + assert compare_objects_as_dicts( + actual_workflow_run, expected_workflow_run, ["id", "created_at", "updated_at"] + ) + + +def test__adapt_github_workflows_to_workflow_runs_given_already_synced_workflow_run_returns_updated_run(): + github_workflow_run = get_github_workflow_run_dict() + org_id = uuid4_str() + repo_id = uuid4_str() + + repo_workflow_run_in_db = get_repo_workflow_run( + repo_workflow_id=repo_id, + provider_workflow_run_id=str(github_workflow_run["id"]), + ) + + class WorkflowRepoService: + def get_repo_workflow_run_by_provider_workflow_run_id(self, *args): + return repo_workflow_run_in_db + + gh_actions_etl_handler = GithubActionsETLHandler(org_id, None, WorkflowRepoService) + actual_workflow_run = ( + gh_actions_etl_handler._adapt_github_workflows_to_workflow_runs( + repo_id, github_workflow_run + ) + ) + + expected_workflow_run = get_repo_workflow_run( + id=repo_workflow_run_in_db.id, + repo_workflow_id=repo_id, + provider_workflow_run_id=str(github_workflow_run["id"]), + event_actor=github_workflow_run["actor"]["login"], + head_branch=github_workflow_run["head_branch"], + status=RepoWorkflowRunsStatus.SUCCESS, + conducted_at=gh_actions_etl_handler._get_datetime_from_gh_datetime( + github_workflow_run["run_started_at"] + ), + duration=gh_actions_etl_handler._get_repo_workflow_run_duration( + github_workflow_run + ), + meta=github_workflow_run, + html_url=github_workflow_run["html_url"], + ) + + assert compare_objects_as_dicts( + actual_workflow_run, expected_workflow_run, ["created_at", "updated_at"] + ) + + +def test__get_repo_workflow_run_duration_given_workflow_run_with_timings_returns_correct_duration(): + repo_workflow_run = get_github_workflow_run_dict( + run_started_at="2021-06-01T12:00:00Z", updated_at="2021-06-01T12:11:00Z" + ) + org_id = uuid4_str() + expected_duration = 660 + gh_actions_etl_handler = GithubActionsETLHandler(org_id, None, None) + actual_duration = gh_actions_etl_handler._get_repo_workflow_run_duration( + repo_workflow_run + ) + assert actual_duration == expected_duration + + +def test__get_repo_workflow_run_duration_given_workflow_run_without_timings_returns_none(): + repo_workflow_run = get_github_workflow_run_dict(run_started_at="", updated_at="") + org_id = uuid4_str() + gh_actions_etl_handler = GithubActionsETLHandler(org_id, None, None) + actual_duration = gh_actions_etl_handler._get_repo_workflow_run_duration( + repo_workflow_run + ) + assert actual_duration is None diff --git a/backend/analytics_server/tests/utilities.py b/backend/analytics_server/tests/utilities.py new file mode 100644 index 000000000..9a101f7c4 --- /dev/null +++ b/backend/analytics_server/tests/utilities.py @@ -0,0 +1,20 @@ +def compare_objects_as_dicts(ob_1, ob_2, ignored_keys=None): + """ + This method can be used to compare between two objects in tests while ignoring keys that are generated as side effects like uuids or autogenerated date time fields. + """ + if not ignored_keys: + ignored_keys = [] + + default_ignored_keys = ["_sa_instance_state"] + final_ignored_keys = set(ignored_keys + default_ignored_keys) + + for key in final_ignored_keys: + if key in ob_1.__dict__: + del ob_1.__dict__[key] + if key in ob_2.__dict__: + del ob_2.__dict__[key] + + if not ob_1.__dict__ == ob_2.__dict__: + print(ob_1.__dict__, "!=", ob_2.__dict__) + return False + return True diff --git a/backend/analytics_server/tests/utils/dict/test_get_average_of_dict_values.py b/backend/analytics_server/tests/utils/dict/test_get_average_of_dict_values.py new file mode 100644 index 000000000..ff6b429d2 --- /dev/null +++ b/backend/analytics_server/tests/utils/dict/test_get_average_of_dict_values.py @@ -0,0 +1,17 @@ +from dora.utils.dict import get_average_of_dict_values + + +def test_empty_dict_returns_zero(): + assert get_average_of_dict_values({}) == 0 + + +def test_nulls_counted_as_zero(): + assert get_average_of_dict_values({"w1": 2, "w2": 4, "w3": None}) == 2 + + +def test_average_of_integers_with_integer_avg(): + assert get_average_of_dict_values({"w1": 2, "w2": 4, "w3": 6}) == 4 + + +def test_average_of_integers_with_decimal_avg_rounded_off(): + assert get_average_of_dict_values({"w1": 2, "w2": 4, "w3": 7}) == 4 diff --git a/backend/analytics_server/tests/utils/dict/test_get_key_to_count_map.py b/backend/analytics_server/tests/utils/dict/test_get_key_to_count_map.py new file mode 100644 index 000000000..e9ace5512 --- /dev/null +++ b/backend/analytics_server/tests/utils/dict/test_get_key_to_count_map.py @@ -0,0 +1,29 @@ +from dora.utils.dict import get_key_to_count_map_from_key_to_list_map + + +def test_empty_dict_return_empty_dict(): + assert get_key_to_count_map_from_key_to_list_map({}) == {} + + +def test_dict_with_list_values(): + assert get_key_to_count_map_from_key_to_list_map( + {"a": [1, 2], "b": ["a", "p", "9"]} + ) == {"a": 2, "b": 3} + + +def test_dict_with_set_values(): + assert get_key_to_count_map_from_key_to_list_map( + {"a": {1, 2}, "b": {"a", "p", "9"}} + ) == {"a": 2, "b": 3} + + +def test_dict_with_non_set_or_list_values(): + assert get_key_to_count_map_from_key_to_list_map( + {"a": None, "b": 0, "c": "Ckk"} + ) == {"a": 0, "b": 0, "c": 0} + + +def test_dict_with_mixed_values(): + assert get_key_to_count_map_from_key_to_list_map( + {"a": None, "b": 0, "c": "Ckk", "e": [1], "g": {"A", "B"}} + ) == {"a": 0, "b": 0, "c": 0, "e": 1, "g": 2} diff --git a/backend/analytics_server/tests/utils/time/test_fill_missing_week_buckets.py b/backend/analytics_server/tests/utils/time/test_fill_missing_week_buckets.py new file mode 100644 index 000000000..48a2f8652 --- /dev/null +++ b/backend/analytics_server/tests/utils/time/test_fill_missing_week_buckets.py @@ -0,0 +1,74 @@ +from dataclasses import dataclass +from datetime import datetime + +import pytz + +from dora.utils.time import Interval, fill_missing_week_buckets + + +last_week_2022 = datetime(2022, 12, 26, 0, 0, 0, tzinfo=pytz.UTC) +first_week_2023 = datetime(2023, 1, 2, 0, 0, 0, tzinfo=pytz.UTC) +second_week_2023 = datetime(2023, 1, 9, 0, 0, 0, tzinfo=pytz.UTC) +third_week_2023 = datetime(2023, 1, 16, 0, 0, 0, tzinfo=pytz.UTC) +fourth_week_2023 = datetime(2023, 1, 23, 0, 0, 0, tzinfo=pytz.UTC) + + +@dataclass +class sample_class: + score: int = 10 + name: str = "MHQ" + + +def test_fill_missing_buckets_fills_missing_weeks_in_middle(): + interval = Interval(last_week_2022, fourth_week_2023) + assert fill_missing_week_buckets( + {last_week_2022: sample_class(1, ""), fourth_week_2023: sample_class(2, "")}, + interval, + ) == { + last_week_2022: sample_class(1, ""), + first_week_2023: None, + second_week_2023: None, + third_week_2023: None, + fourth_week_2023: sample_class(2, ""), + } + + +def test_fill_missing_buckets_fills_missing_weeks_in_past(): + interval = Interval(last_week_2022, fourth_week_2023) + assert fill_missing_week_buckets( + {third_week_2023: sample_class(1, ""), fourth_week_2023: sample_class(2, "")}, + interval, + ) == { + last_week_2022: None, + first_week_2023: None, + second_week_2023: None, + third_week_2023: sample_class(1, ""), + fourth_week_2023: sample_class(2, ""), + } + + +def test_fill_missing_buckets_fills_missing_weeks_in_future(): + interval = Interval(last_week_2022, fourth_week_2023) + assert fill_missing_week_buckets( + {last_week_2022: sample_class(1, ""), first_week_2023: sample_class(2, "")}, + interval, + ) == { + last_week_2022: sample_class(1, ""), + first_week_2023: sample_class(2, ""), + second_week_2023: None, + third_week_2023: None, + fourth_week_2023: None, + } + + +def test_fill_missing_buckets_fills_past_and_future_weeks_with_callable(): + interval = Interval(last_week_2022, fourth_week_2023) + assert fill_missing_week_buckets( + {first_week_2023: sample_class(2, "")}, interval, sample_class + ) == { + last_week_2022: sample_class(), + first_week_2023: sample_class(2, ""), + second_week_2023: sample_class(), + third_week_2023: sample_class(), + fourth_week_2023: sample_class(), + } diff --git a/backend/analytics_server/tests/utils/time/test_generate_expanded_buckets.py b/backend/analytics_server/tests/utils/time/test_generate_expanded_buckets.py new file mode 100644 index 000000000..95d3706f9 --- /dev/null +++ b/backend/analytics_server/tests/utils/time/test_generate_expanded_buckets.py @@ -0,0 +1,288 @@ +from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Dict, List, Any + +import pytest +import pytz + +from dora.utils.time import ( + Interval, + generate_expanded_buckets, + get_given_weeks_monday, + time_now, +) + + +@dataclass +class AnyObject: + state_changed_at: datetime + + +def test_incorrect_interval_raises_exception(): + object_list = [] + from_time = time_now() + to_time = from_time - timedelta(seconds=1) + + attribute = "state_changed_at" + with pytest.raises(AssertionError) as e: + buckets = generate_expanded_buckets( + object_list, Interval(from_time, to_time), attribute + ) + assert ( + str(e.value) + == f"from_time: {from_time.isoformat()} is greater than to_time: {to_time.isoformat()}" + ) + + +def test_missing_attribute_raise_exception(): + object_list = [AnyObject(time_now())] + from_time = time_now() + to_time = from_time + timedelta(seconds=1) + attribute = "updated_at" + with pytest.raises(AttributeError) as e: + buckets = generate_expanded_buckets( + object_list, Interval(from_time, to_time), attribute + ) + + +def test_incorrect_attribute_type_raise_exception(): + object_list = [AnyObject("hello")] + from_time = time_now() + to_time = from_time + timedelta(seconds=1) + attribute = "state_changed_at" + with pytest.raises(Exception) as e: + buckets = generate_expanded_buckets( + object_list, Interval(from_time, to_time), attribute + ) + assert ( + str(e.value) + == f"Type of datetime_attribute:{type(getattr(object_list[0], attribute))} is not datetime" + ) + + +def test_empty_data_generates_correct_buckets(): + object_list = [] + from_time = time_now() - timedelta(days=10) + to_time = from_time + timedelta(seconds=1) + attribute = "state_changed_at" + + ans_buckets = defaultdict(list) + + curr_date = get_given_weeks_monday(from_time) + + while curr_date < to_time: + ans_buckets[curr_date] = [] + curr_date = curr_date + timedelta(days=7) + + assert ans_buckets == generate_expanded_buckets( + object_list, Interval(from_time, to_time), attribute + ) + + +def test_data_generates_empty_middle_buckets(): + first_week_2023 = datetime(2023, 1, 2, 0, 0, 0, tzinfo=pytz.UTC) + second_week_2023 = datetime(2023, 1, 9, 0, 0, 0, tzinfo=pytz.UTC) + third_week_2023 = datetime(2023, 1, 16, 0, 0, 0, tzinfo=pytz.UTC) + + from_time = first_week_2023 + timedelta(days=1) + to_time = third_week_2023 + timedelta(days=5) + + obj1 = AnyObject(first_week_2023 + timedelta(days=2)) + obj2 = AnyObject(first_week_2023 + timedelta(days=3)) + obj3 = AnyObject(first_week_2023 + timedelta(days=4)) + obj4 = AnyObject(third_week_2023 + timedelta(days=2)) + obj5 = AnyObject(third_week_2023 + timedelta(days=3)) + obj6 = AnyObject(third_week_2023 + timedelta(days=4)) + object_list = [obj1, obj2, obj3, obj4, obj5, obj6] + + attribute = "state_changed_at" + + ans_buckets: Dict[datetime, List[Any]] = defaultdict(list) + + ans_buckets[first_week_2023] = [obj1, obj2, obj3] + ans_buckets[second_week_2023] = [] + ans_buckets[third_week_2023] = [obj4, obj5, obj6] + + curr_date = get_given_weeks_monday(from_time) + + assert ans_buckets == generate_expanded_buckets( + object_list, Interval(from_time, to_time), attribute + ) + + +def test_data_within_interval_generates_correctly_filled_buckets(): + first_week_2023 = datetime(2023, 1, 2, 0, 0, 0, tzinfo=pytz.UTC) + second_week_2023 = datetime(2023, 1, 9, 0, 0, 0, tzinfo=pytz.UTC) + third_week_2023 = datetime(2023, 1, 16, 0, 0, 0, tzinfo=pytz.UTC) + + from_time = first_week_2023 + timedelta(days=1) + to_time = third_week_2023 + timedelta(days=5) + + obj1 = AnyObject(first_week_2023 + timedelta(days=2)) + obj2 = AnyObject(first_week_2023 + timedelta(days=3)) + obj3 = AnyObject(first_week_2023 + timedelta(days=4)) + obj4 = AnyObject(second_week_2023) + obj5 = AnyObject(second_week_2023 + timedelta(days=6)) + obj6 = AnyObject(third_week_2023 + timedelta(days=4)) + obj7 = AnyObject(third_week_2023 + timedelta(days=2)) + obj8 = AnyObject(third_week_2023 + timedelta(days=3)) + obj9 = AnyObject(third_week_2023 + timedelta(days=4)) + object_list = [obj1, obj2, obj3, obj4, obj5, obj6, obj7, obj8, obj9] + + attribute = "state_changed_at" + + ans_buckets = defaultdict(list) + + ans_buckets[first_week_2023] = [obj1, obj2, obj3] + ans_buckets[second_week_2023] = [obj4, obj5] + ans_buckets[third_week_2023] = [obj6, obj7, obj8, obj9] + + assert ans_buckets == generate_expanded_buckets( + object_list, Interval(from_time, to_time), attribute + ) + + +def test_data_outside_interval_generates_correctly_filled_buckets(): + last_week_2022 = datetime(2022, 12, 26, 0, 0, 0, tzinfo=pytz.UTC) + first_week_2023 = datetime(2023, 1, 2, 0, 0, 0, tzinfo=pytz.UTC) + second_week_2023 = datetime(2023, 1, 9, 0, 0, 0, tzinfo=pytz.UTC) + third_week_2023 = datetime(2023, 1, 16, 0, 0, 0, tzinfo=pytz.UTC) + fourth_week_2023 = datetime(2023, 1, 23, 0, 0, 0, tzinfo=pytz.UTC) + + from_time = first_week_2023 + timedelta(days=1) + to_time = third_week_2023 + timedelta(days=5) + + obj1 = AnyObject(last_week_2022 + timedelta(days=2)) + obj2 = AnyObject(last_week_2022 + timedelta(days=3)) + obj3 = AnyObject(last_week_2022 + timedelta(days=4)) + obj4 = AnyObject(last_week_2022) + obj5 = AnyObject(second_week_2023 + timedelta(days=6)) + obj6 = AnyObject(fourth_week_2023 + timedelta(days=4)) + obj7 = AnyObject(fourth_week_2023 + timedelta(days=2)) + obj8 = AnyObject(fourth_week_2023 + timedelta(days=3)) + obj9 = AnyObject(fourth_week_2023 + timedelta(days=4)) + object_list = [obj1, obj2, obj3, obj4, obj5, obj6, obj7, obj8, obj9] + + attribute = "state_changed_at" + + ans_buckets = defaultdict(list) + + ans_buckets[last_week_2022] = [obj1, obj2, obj3, obj4] + ans_buckets[first_week_2023] = [] + ans_buckets[second_week_2023] = [obj5] + ans_buckets[third_week_2023] = [] + ans_buckets[fourth_week_2023] = [obj6, obj7, obj8, obj9] + + assert ans_buckets == generate_expanded_buckets( + object_list, Interval(from_time, to_time), attribute + ) + + +def test_daily_buckets_with_one_per_day(): + object_list = [ + AnyObject(datetime(2022, 1, 1, tzinfo=pytz.UTC)), + AnyObject(datetime(2022, 1, 2, tzinfo=pytz.UTC)), + AnyObject(datetime(2022, 1, 2, tzinfo=pytz.UTC)), + ] + from_time = datetime(2022, 1, 1, tzinfo=pytz.UTC) + to_time = datetime(2022, 1, 3, tzinfo=pytz.UTC) + attribute = "state_changed_at" + granularity = "daily" + ans_buckets = defaultdict(list) + ans_buckets[datetime.fromisoformat("2022-01-01T00:00:00+00:00")] = [object_list[0]] + ans_buckets[datetime.fromisoformat("2022-01-02T00:00:00+00:00")] = object_list[1:] + ans_buckets[datetime.fromisoformat("2022-01-03T00:00:00+00:00")] = [] + assert ans_buckets == generate_expanded_buckets( + object_list, Interval(from_time, to_time), attribute, granularity + ) + + +def test_daily_buckets_with_multiple_per_day(): + object_list_2 = [ + AnyObject(datetime(2022, 1, 1, tzinfo=pytz.UTC)), + AnyObject(datetime(2022, 1, 2, tzinfo=pytz.UTC)), + AnyObject(datetime(2022, 1, 2, 12, 30, tzinfo=pytz.UTC)), + ] + from_time_2 = datetime(2022, 1, 1, tzinfo=pytz.UTC) + to_time_2 = datetime(2022, 1, 3, tzinfo=pytz.UTC) + attribute = "state_changed_at" + granularity = "daily" + ans_buckets_2 = defaultdict(list) + ans_buckets_2[datetime.fromisoformat("2022-01-01T00:00:00+00:00")] = [ + object_list_2[0] + ] + ans_buckets_2[datetime.fromisoformat("2022-01-02T00:00:00+00:00")] = [ + object_list_2[1], + object_list_2[2], + ] + ans_buckets_2[datetime.fromisoformat("2022-01-03T00:00:00+00:00")] = [] + assert ans_buckets_2 == generate_expanded_buckets( + object_list_2, Interval(from_time_2, to_time_2), attribute, granularity + ) + + +def test_daily_buckets_without_objects(): + object_list_3 = [] + from_time_3 = datetime(2022, 1, 1, tzinfo=pytz.UTC) + to_time_3 = datetime(2022, 1, 3, tzinfo=pytz.UTC) + attribute = "state_changed_at" + granularity = "daily" + ans_buckets_3 = defaultdict(list) + ans_buckets_3[datetime.fromisoformat("2022-01-01T00:00:00+00:00")] = [] + ans_buckets_3[datetime.fromisoformat("2022-01-02T00:00:00+00:00")] = [] + ans_buckets_3[datetime.fromisoformat("2022-01-03T00:00:00+00:00")] = [] + assert ans_buckets_3 == generate_expanded_buckets( + object_list_3, Interval(from_time_3, to_time_3), attribute, granularity + ) + + +def test_monthly_buckets(): + object_list = [ + AnyObject(datetime(2022, 1, 1, tzinfo=pytz.UTC)), + AnyObject(datetime(2022, 2, 1, tzinfo=pytz.UTC)), + AnyObject(datetime(2022, 2, 15, tzinfo=pytz.UTC)), + AnyObject(datetime(2022, 3, 1, tzinfo=pytz.UTC)), + AnyObject(datetime(2022, 3, 15, tzinfo=pytz.UTC)), + ] + from_time = datetime(2022, 1, 1, tzinfo=pytz.UTC) + to_time = datetime(2022, 3, 15, tzinfo=pytz.UTC) + attribute = "state_changed_at" + granularity = "monthly" + ans_buckets = defaultdict(list) + ans_buckets[datetime.fromisoformat("2022-01-01T00:00:00+00:00")] = [object_list[0]] + ans_buckets[datetime.fromisoformat("2022-02-01T00:00:00+00:00")] = object_list[1:3] + ans_buckets[datetime.fromisoformat("2022-03-01T00:00:00+00:00")] = object_list[3:] + assert ans_buckets == generate_expanded_buckets( + object_list, Interval(from_time, to_time), attribute, granularity + ) + + +def test_data_generates_empty_middle_buckets_for_monthly(): + first_month_2023 = datetime(2023, 1, 2, 0, 0, 0, tzinfo=pytz.UTC) + second_month_2023 = datetime(2023, 2, 9, 0, 0, 0, tzinfo=pytz.UTC) + third_month_2023 = datetime(2023, 3, 16, 0, 0, 0, tzinfo=pytz.UTC) + + from_time = first_month_2023 + timedelta(days=1) + to_time = third_month_2023 + timedelta(days=5) + + obj1 = AnyObject(first_month_2023 + timedelta(days=2)) + obj2 = AnyObject(first_month_2023 + timedelta(days=3)) + obj3 = AnyObject(first_month_2023 + timedelta(days=4)) + obj4 = AnyObject(third_month_2023 + timedelta(days=2)) + obj5 = AnyObject(third_month_2023 + timedelta(days=3)) + obj6 = AnyObject(third_month_2023 + timedelta(days=4)) + object_list = [obj1, obj2, obj3, obj4, obj5, obj6] + + attribute = "state_changed_at" + granularity = "monthly" + + ans_buckets: Dict[datetime, List[Any]] = defaultdict(list) + + ans_buckets[first_month_2023.replace(day=1)] = [obj1, obj2, obj3] + ans_buckets[second_month_2023.replace(day=1)] = [] + ans_buckets[third_month_2023.replace(day=1)] = [obj4, obj5, obj6] + + assert ans_buckets == generate_expanded_buckets( + object_list, Interval(from_time, to_time), attribute, granularity + ) diff --git a/apiserver/dev-requirements.txt b/backend/dev-requirements.txt similarity index 100% rename from apiserver/dev-requirements.txt rename to backend/dev-requirements.txt diff --git a/apiserver/env.example b/backend/env.example similarity index 100% rename from apiserver/env.example rename to backend/env.example diff --git a/apiserver/requirements.txt b/backend/requirements.txt similarity index 100% rename from apiserver/requirements.txt rename to backend/requirements.txt From 5cde05702c3e636e57af10946b25e05d45825b35 Mon Sep 17 00:00:00 2001 From: amoghjalan Date: Mon, 22 Apr 2024 16:01:35 +0530 Subject: [PATCH 2/2] Remove older code --- apiserver/app.py | 34 -- apiserver/dora/__init__.py | 0 apiserver/dora/api/__init__.py | 0 apiserver/dora/api/deployment_analytics.py | 216 -------- apiserver/dora/api/hello.py | 9 - apiserver/dora/api/incidents.py | 250 --------- apiserver/dora/api/integrations.py | 95 ---- apiserver/dora/api/pull_requests.py | 169 ------ apiserver/dora/api/request_utils.py | 77 --- apiserver/dora/api/resources/__init__.py | 0 apiserver/dora/api/resources/code_resouces.py | 107 ---- .../dora/api/resources/core_resources.py | 33 -- .../api/resources/deployment_resources.py | 38 -- .../dora/api/resources/incident_resources.py | 71 --- .../dora/api/resources/settings_resource.py | 61 --- apiserver/dora/api/settings.py | 172 ------ apiserver/dora/api/sync.py | 12 - apiserver/dora/api/teams.py | 79 --- apiserver/dora/config/config.ini | 3 - apiserver/dora/exapi/__init__.py | 0 apiserver/dora/exapi/git_incidents.py | 74 --- apiserver/dora/exapi/github.py | 254 --------- apiserver/dora/exapi/models/__init__.py | 0 apiserver/dora/exapi/models/git_incidents.py | 12 - apiserver/dora/exapi/models/github.py | 45 -- apiserver/dora/service/__init__.py | 0 apiserver/dora/service/code/__init__.py | 3 - apiserver/dora/service/code/integration.py | 27 - apiserver/dora/service/code/lead_time.py | 264 --------- .../dora/service/code/models/lead_time.py | 49 -- apiserver/dora/service/code/pr_filter.py | 113 ---- apiserver/dora/service/code/sync/__init__.py | 1 - .../service/code/sync/etl_code_analytics.py | 173 ------ .../service/code/sync/etl_code_factory.py | 13 - .../service/code/sync/etl_github_handler.py | 373 ------------- .../dora/service/code/sync/etl_handler.py | 125 ----- .../service/code/sync/etl_provider_handler.py | 53 -- apiserver/dora/service/code/sync/models.py | 19 - .../code/sync/revert_prs_github_sync.py | 185 ------- apiserver/dora/service/core/teams.py | 35 -- .../dora/service/deployments/__init__.py | 1 - .../dora/service/deployments/analytics.py | 257 --------- .../deployments/deployment_pr_mapper.py | 79 --- .../service/deployments/deployment_service.py | 171 ------ .../deployments_factory_service.py | 46 -- apiserver/dora/service/deployments/factory.py | 27 - .../service/deployments/models/__init__.py | 0 .../service/deployments/models/adapter.py | 105 ---- .../dora/service/deployments/models/models.py | 46 -- .../deployments/pr_deployments_service.py | 51 -- .../workflow_deployments_service.py | 92 ---- .../service/external_integrations_service.py | 63 --- apiserver/dora/service/incidents/__init__.py | 1 - .../dora/service/incidents/incident_filter.py | 109 ---- apiserver/dora/service/incidents/incidents.py | 213 -------- .../dora/service/incidents/integration.py | 65 --- .../incidents/models/mean_time_to_recovery.py | 34 -- .../dora/service/incidents/sync/__init__.py | 1 - .../sync/etl_git_incidents_handler.py | 242 --------- .../service/incidents/sync/etl_handler.py | 107 ---- .../incidents/sync/etl_incidents_factory.py | 15 - .../incidents/sync/etl_provider_handler.py | 43 -- .../merge_to_deploy_broker/__init__.py | 2 - .../merge_to_deploy_broker/mtd_handler.py | 126 ----- .../service/merge_to_deploy_broker/utils.py | 49 -- apiserver/dora/service/pr_analytics.py | 22 - apiserver/dora/service/query_validator.py | 77 --- apiserver/dora/service/settings/__init__.py | 2 - .../settings/configuration_settings.py | 376 ------------- .../service/settings/default_settings_data.py | 29 - apiserver/dora/service/settings/models.py | 41 -- .../settings/setting_type_validator.py | 19 - apiserver/dora/service/sync_data.py | 34 -- apiserver/dora/service/workflows/__init__.py | 1 - .../dora/service/workflows/integration.py | 28 - .../dora/service/workflows/sync/__init__.py | 1 - .../sync/etl_github_actions_handler.py | 189 ------- .../service/workflows/sync/etl_handler.py | 134 ----- .../workflows/sync/etl_provider_handler.py | 36 -- .../workflows/sync/etl_workflows_factory.py | 15 - .../dora/service/workflows/workflow_filter.py | 52 -- apiserver/dora/store/__init__.py | 36 -- apiserver/dora/store/initialise_db.py | 27 - apiserver/dora/store/models/__init__.py | 7 - apiserver/dora/store/models/code/__init__.py | 31 -- apiserver/dora/store/models/code/enums.py | 35 -- apiserver/dora/store/models/code/filter.py | 98 ---- .../dora/store/models/code/pull_requests.py | 155 ------ .../dora/store/models/code/repository.py | 108 ---- .../store/models/code/workflows/__init__.py | 3 - .../dora/store/models/code/workflows/enums.py | 32 -- .../store/models/code/workflows/filter.py | 81 --- .../store/models/code/workflows/workflows.py | 62 --- apiserver/dora/store/models/core/__init__.py | 3 - .../dora/store/models/core/organization.py | 23 - apiserver/dora/store/models/core/teams.py | 26 - apiserver/dora/store/models/core/users.py | 21 - .../dora/store/models/incidents/__init__.py | 15 - .../dora/store/models/incidents/enums.py | 35 -- .../dora/store/models/incidents/filter.py | 45 -- .../dora/store/models/incidents/incidents.py | 59 -- .../dora/store/models/incidents/services.py | 45 -- .../store/models/integrations/__init__.py | 2 - .../dora/store/models/integrations/enums.py | 12 - .../store/models/integrations/integrations.py | 49 -- .../dora/store/models/settings/__init__.py | 2 - .../models/settings/configuration_settings.py | 33 -- apiserver/dora/store/models/settings/enums.py | 7 - apiserver/dora/store/repos/__init__.py | 0 apiserver/dora/store/repos/code.py | 333 ------------ apiserver/dora/store/repos/core.py | 108 ---- apiserver/dora/store/repos/incidents.py | 148 ----- apiserver/dora/store/repos/integrations.py | 2 - apiserver/dora/store/repos/settings.py | 90 ---- apiserver/dora/store/repos/workflows.py | 227 -------- apiserver/dora/utils/__init__.py | 0 apiserver/dora/utils/cryptography.py | 95 ---- apiserver/dora/utils/dict.py | 32 -- apiserver/dora/utils/github.py | 50 -- apiserver/dora/utils/lock.py | 41 -- apiserver/dora/utils/log.py | 17 - apiserver/dora/utils/regex.py | 29 - apiserver/dora/utils/string.py | 5 - apiserver/dora/utils/time.py | 270 ---------- apiserver/tests/__init__.py | 0 apiserver/tests/factories/__init__.py | 0 apiserver/tests/factories/models/__init__.py | 12 - apiserver/tests/factories/models/code.py | 201 ------- .../tests/factories/models/exapi/__init__.py | 0 .../tests/factories/models/exapi/github.py | 159 ------ apiserver/tests/factories/models/incidents.py | 93 ---- .../tests/service/Incidents/sync/__init__.py | 0 .../sync/test_etl_git_incidents_handler.py | 141 ----- .../Incidents/test_change_failure_rate.py | 352 ------------ .../test_deployment_incident_mapper.py | 119 ----- apiserver/tests/service/__init__.py | 0 apiserver/tests/service/code/__init__.py | 0 apiserver/tests/service/code/sync/__init__.py | 0 .../code/sync/test_etl_code_analytics.py | 505 ------------------ .../code/sync/test_etl_github_handler.py | 353 ------------ .../service/code/test_lead_time_service.py | 198 ------- .../tests/service/deployments/__init__.py | 0 .../deployments/test_deployment_frequency.py | 181 ------- .../deployments/test_deployment_pr_mapper.py | 183 ------- apiserver/tests/service/workflows/__init__.py | 0 .../tests/service/workflows/sync/__init__.py | 0 .../sync/test_etl_github_actions_handler.py | 110 ---- apiserver/tests/utilities.py | 20 - .../dict/test_get_average_of_dict_values.py | 17 - .../utils/dict/test_get_key_to_count_map.py | 29 - .../time/test_fill_missing_week_buckets.py | 74 --- .../time/test_generate_expanded_buckets.py | 288 ---------- 152 files changed, 11779 deletions(-) delete mode 100644 apiserver/app.py delete mode 100644 apiserver/dora/__init__.py delete mode 100644 apiserver/dora/api/__init__.py delete mode 100644 apiserver/dora/api/deployment_analytics.py delete mode 100644 apiserver/dora/api/hello.py delete mode 100644 apiserver/dora/api/incidents.py delete mode 100644 apiserver/dora/api/integrations.py delete mode 100644 apiserver/dora/api/pull_requests.py delete mode 100644 apiserver/dora/api/request_utils.py delete mode 100644 apiserver/dora/api/resources/__init__.py delete mode 100644 apiserver/dora/api/resources/code_resouces.py delete mode 100644 apiserver/dora/api/resources/core_resources.py delete mode 100644 apiserver/dora/api/resources/deployment_resources.py delete mode 100644 apiserver/dora/api/resources/incident_resources.py delete mode 100644 apiserver/dora/api/resources/settings_resource.py delete mode 100644 apiserver/dora/api/settings.py delete mode 100644 apiserver/dora/api/sync.py delete mode 100644 apiserver/dora/api/teams.py delete mode 100644 apiserver/dora/config/config.ini delete mode 100644 apiserver/dora/exapi/__init__.py delete mode 100644 apiserver/dora/exapi/git_incidents.py delete mode 100644 apiserver/dora/exapi/github.py delete mode 100644 apiserver/dora/exapi/models/__init__.py delete mode 100644 apiserver/dora/exapi/models/git_incidents.py delete mode 100644 apiserver/dora/exapi/models/github.py delete mode 100644 apiserver/dora/service/__init__.py delete mode 100644 apiserver/dora/service/code/__init__.py delete mode 100644 apiserver/dora/service/code/integration.py delete mode 100644 apiserver/dora/service/code/lead_time.py delete mode 100644 apiserver/dora/service/code/models/lead_time.py delete mode 100644 apiserver/dora/service/code/pr_filter.py delete mode 100644 apiserver/dora/service/code/sync/__init__.py delete mode 100644 apiserver/dora/service/code/sync/etl_code_analytics.py delete mode 100644 apiserver/dora/service/code/sync/etl_code_factory.py delete mode 100644 apiserver/dora/service/code/sync/etl_github_handler.py delete mode 100644 apiserver/dora/service/code/sync/etl_handler.py delete mode 100644 apiserver/dora/service/code/sync/etl_provider_handler.py delete mode 100644 apiserver/dora/service/code/sync/models.py delete mode 100644 apiserver/dora/service/code/sync/revert_prs_github_sync.py delete mode 100644 apiserver/dora/service/core/teams.py delete mode 100644 apiserver/dora/service/deployments/__init__.py delete mode 100644 apiserver/dora/service/deployments/analytics.py delete mode 100644 apiserver/dora/service/deployments/deployment_pr_mapper.py delete mode 100644 apiserver/dora/service/deployments/deployment_service.py delete mode 100644 apiserver/dora/service/deployments/deployments_factory_service.py delete mode 100644 apiserver/dora/service/deployments/factory.py delete mode 100644 apiserver/dora/service/deployments/models/__init__.py delete mode 100644 apiserver/dora/service/deployments/models/adapter.py delete mode 100644 apiserver/dora/service/deployments/models/models.py delete mode 100644 apiserver/dora/service/deployments/pr_deployments_service.py delete mode 100644 apiserver/dora/service/deployments/workflow_deployments_service.py delete mode 100644 apiserver/dora/service/external_integrations_service.py delete mode 100644 apiserver/dora/service/incidents/__init__.py delete mode 100644 apiserver/dora/service/incidents/incident_filter.py delete mode 100644 apiserver/dora/service/incidents/incidents.py delete mode 100644 apiserver/dora/service/incidents/integration.py delete mode 100644 apiserver/dora/service/incidents/models/mean_time_to_recovery.py delete mode 100644 apiserver/dora/service/incidents/sync/__init__.py delete mode 100644 apiserver/dora/service/incidents/sync/etl_git_incidents_handler.py delete mode 100644 apiserver/dora/service/incidents/sync/etl_handler.py delete mode 100644 apiserver/dora/service/incidents/sync/etl_incidents_factory.py delete mode 100644 apiserver/dora/service/incidents/sync/etl_provider_handler.py delete mode 100644 apiserver/dora/service/merge_to_deploy_broker/__init__.py delete mode 100644 apiserver/dora/service/merge_to_deploy_broker/mtd_handler.py delete mode 100644 apiserver/dora/service/merge_to_deploy_broker/utils.py delete mode 100644 apiserver/dora/service/pr_analytics.py delete mode 100644 apiserver/dora/service/query_validator.py delete mode 100644 apiserver/dora/service/settings/__init__.py delete mode 100644 apiserver/dora/service/settings/configuration_settings.py delete mode 100644 apiserver/dora/service/settings/default_settings_data.py delete mode 100644 apiserver/dora/service/settings/models.py delete mode 100644 apiserver/dora/service/settings/setting_type_validator.py delete mode 100644 apiserver/dora/service/sync_data.py delete mode 100644 apiserver/dora/service/workflows/__init__.py delete mode 100644 apiserver/dora/service/workflows/integration.py delete mode 100644 apiserver/dora/service/workflows/sync/__init__.py delete mode 100644 apiserver/dora/service/workflows/sync/etl_github_actions_handler.py delete mode 100644 apiserver/dora/service/workflows/sync/etl_handler.py delete mode 100644 apiserver/dora/service/workflows/sync/etl_provider_handler.py delete mode 100644 apiserver/dora/service/workflows/sync/etl_workflows_factory.py delete mode 100644 apiserver/dora/service/workflows/workflow_filter.py delete mode 100644 apiserver/dora/store/__init__.py delete mode 100644 apiserver/dora/store/initialise_db.py delete mode 100644 apiserver/dora/store/models/__init__.py delete mode 100644 apiserver/dora/store/models/code/__init__.py delete mode 100644 apiserver/dora/store/models/code/enums.py delete mode 100644 apiserver/dora/store/models/code/filter.py delete mode 100644 apiserver/dora/store/models/code/pull_requests.py delete mode 100644 apiserver/dora/store/models/code/repository.py delete mode 100644 apiserver/dora/store/models/code/workflows/__init__.py delete mode 100644 apiserver/dora/store/models/code/workflows/enums.py delete mode 100644 apiserver/dora/store/models/code/workflows/filter.py delete mode 100644 apiserver/dora/store/models/code/workflows/workflows.py delete mode 100644 apiserver/dora/store/models/core/__init__.py delete mode 100644 apiserver/dora/store/models/core/organization.py delete mode 100644 apiserver/dora/store/models/core/teams.py delete mode 100644 apiserver/dora/store/models/core/users.py delete mode 100644 apiserver/dora/store/models/incidents/__init__.py delete mode 100644 apiserver/dora/store/models/incidents/enums.py delete mode 100644 apiserver/dora/store/models/incidents/filter.py delete mode 100644 apiserver/dora/store/models/incidents/incidents.py delete mode 100644 apiserver/dora/store/models/incidents/services.py delete mode 100644 apiserver/dora/store/models/integrations/__init__.py delete mode 100644 apiserver/dora/store/models/integrations/enums.py delete mode 100644 apiserver/dora/store/models/integrations/integrations.py delete mode 100644 apiserver/dora/store/models/settings/__init__.py delete mode 100644 apiserver/dora/store/models/settings/configuration_settings.py delete mode 100644 apiserver/dora/store/models/settings/enums.py delete mode 100644 apiserver/dora/store/repos/__init__.py delete mode 100644 apiserver/dora/store/repos/code.py delete mode 100644 apiserver/dora/store/repos/core.py delete mode 100644 apiserver/dora/store/repos/incidents.py delete mode 100644 apiserver/dora/store/repos/integrations.py delete mode 100644 apiserver/dora/store/repos/settings.py delete mode 100644 apiserver/dora/store/repos/workflows.py delete mode 100644 apiserver/dora/utils/__init__.py delete mode 100644 apiserver/dora/utils/cryptography.py delete mode 100644 apiserver/dora/utils/dict.py delete mode 100644 apiserver/dora/utils/github.py delete mode 100644 apiserver/dora/utils/lock.py delete mode 100644 apiserver/dora/utils/log.py delete mode 100644 apiserver/dora/utils/regex.py delete mode 100644 apiserver/dora/utils/string.py delete mode 100644 apiserver/dora/utils/time.py delete mode 100644 apiserver/tests/__init__.py delete mode 100644 apiserver/tests/factories/__init__.py delete mode 100644 apiserver/tests/factories/models/__init__.py delete mode 100644 apiserver/tests/factories/models/code.py delete mode 100644 apiserver/tests/factories/models/exapi/__init__.py delete mode 100644 apiserver/tests/factories/models/exapi/github.py delete mode 100644 apiserver/tests/factories/models/incidents.py delete mode 100644 apiserver/tests/service/Incidents/sync/__init__.py delete mode 100644 apiserver/tests/service/Incidents/sync/test_etl_git_incidents_handler.py delete mode 100644 apiserver/tests/service/Incidents/test_change_failure_rate.py delete mode 100644 apiserver/tests/service/Incidents/test_deployment_incident_mapper.py delete mode 100644 apiserver/tests/service/__init__.py delete mode 100644 apiserver/tests/service/code/__init__.py delete mode 100644 apiserver/tests/service/code/sync/__init__.py delete mode 100644 apiserver/tests/service/code/sync/test_etl_code_analytics.py delete mode 100644 apiserver/tests/service/code/sync/test_etl_github_handler.py delete mode 100644 apiserver/tests/service/code/test_lead_time_service.py delete mode 100644 apiserver/tests/service/deployments/__init__.py delete mode 100644 apiserver/tests/service/deployments/test_deployment_frequency.py delete mode 100644 apiserver/tests/service/deployments/test_deployment_pr_mapper.py delete mode 100644 apiserver/tests/service/workflows/__init__.py delete mode 100644 apiserver/tests/service/workflows/sync/__init__.py delete mode 100644 apiserver/tests/service/workflows/sync/test_etl_github_actions_handler.py delete mode 100644 apiserver/tests/utilities.py delete mode 100644 apiserver/tests/utils/dict/test_get_average_of_dict_values.py delete mode 100644 apiserver/tests/utils/dict/test_get_key_to_count_map.py delete mode 100644 apiserver/tests/utils/time/test_fill_missing_week_buckets.py delete mode 100644 apiserver/tests/utils/time/test_generate_expanded_buckets.py diff --git a/apiserver/app.py b/apiserver/app.py deleted file mode 100644 index e310eee9f..000000000 --- a/apiserver/app.py +++ /dev/null @@ -1,34 +0,0 @@ -from flask import Flask - -from dora.store import configure_db_with_app -from env import load_app_env - -load_app_env() - -from dora.api.hello import app as core_api -from dora.api.settings import app as settings_api -from dora.api.pull_requests import app as pull_requests_api -from dora.api.incidents import app as incidents_api -from dora.api.integrations import app as integrations_api -from dora.api.deployment_analytics import app as deployment_analytics_api -from dora.api.teams import app as teams_api -from dora.api.sync import app as sync_api - -from dora.store.initialise_db import initialize_database - -app = Flask(__name__) - -app.register_blueprint(core_api) -app.register_blueprint(settings_api) -app.register_blueprint(pull_requests_api) -app.register_blueprint(incidents_api) -app.register_blueprint(deployment_analytics_api) -app.register_blueprint(integrations_api) -app.register_blueprint(teams_api) -app.register_blueprint(sync_api) - -configure_db_with_app(app) -initialize_database(app) - -if __name__ == "__main__": - app.run() diff --git a/apiserver/dora/__init__.py b/apiserver/dora/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/dora/api/__init__.py b/apiserver/dora/api/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/dora/api/deployment_analytics.py b/apiserver/dora/api/deployment_analytics.py deleted file mode 100644 index f6d82461b..000000000 --- a/apiserver/dora/api/deployment_analytics.py +++ /dev/null @@ -1,216 +0,0 @@ -from werkzeug.exceptions import NotFound -from collections import defaultdict -from typing import Dict, List -from datetime import datetime -import json - -from flask import Blueprint -from voluptuous import Required, Schema, Coerce, All, Optional -from dora.api.resources.code_resouces import get_non_paginated_pr_response -from dora.service.deployments.deployments_factory_service import ( - DeploymentsFactoryService, -) -from dora.service.deployments.factory import get_deployments_factory -from dora.service.pr_analytics import get_pr_analytics_service -from dora.service.code.pr_filter import apply_pr_filter - -from dora.api.request_utils import coerce_workflow_filter, queryschema -from dora.api.resources.deployment_resources import ( - adapt_deployment, - adapt_deployment_frequency_metrics, -) -from dora.service.deployments.analytics import get_deployment_analytics_service -from dora.service.query_validator import get_query_validator -from dora.store.models import SettingType, EntityType, Team -from dora.store.models.code.filter import PRFilter -from dora.store.models.code.pull_requests import PullRequest -from dora.store.models.code.repository import OrgRepo, TeamRepos -from dora.store.models.code.workflows.filter import WorkflowFilter -from dora.service.deployments.models.models import ( - Deployment, - DeploymentFrequencyMetrics, - DeploymentType, -) -from dora.store.repos.code import CodeRepoService - - -app = Blueprint("deployment_analytics", __name__) - - -@app.route("/teams//deployment_analytics", methods={"GET"}) -@queryschema( - Schema( - { - Required("from_time"): All(str, Coerce(datetime.fromisoformat)), - Required("to_time"): All(str, Coerce(datetime.fromisoformat)), - Optional("pr_filter"): All(str, Coerce(json.loads)), - Optional("workflow_filter"): All(str, Coerce(coerce_workflow_filter)), - } - ), -) -def get_team_deployment_analytics( - team_id: str, - from_time: datetime, - to_time: datetime, - pr_filter: Dict = None, - workflow_filter: WorkflowFilter = None, -): - query_validator = get_query_validator() - interval = query_validator.interval_validator(from_time, to_time) - query_validator.team_validator(team_id) - - pr_filter: PRFilter = apply_pr_filter( - pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] - ) - code_repo_service = CodeRepoService() - - team_repos: List[TeamRepos] = code_repo_service.get_active_team_repos_by_team_id( - team_id - ) - org_repos: List[OrgRepo] = code_repo_service.get_active_org_repos_by_ids( - [str(team_repo.org_repo_id) for team_repo in team_repos] - ) - - deployments_analytics_service = get_deployment_analytics_service() - - repo_id_to_deployments_map_with_prs: Dict[ - str, List[Dict[Deployment, List[PullRequest]]] - ] = deployments_analytics_service.get_team_successful_deployments_in_interval_with_related_prs( - team_id, interval, pr_filter, workflow_filter - ) - - repo_id_deployments_map = defaultdict(list) - - for repo_id, deployment_to_prs_map in repo_id_to_deployments_map_with_prs.items(): - adapted_deployments = [] - for deployment, prs in deployment_to_prs_map.items(): - adapted_deployment = adapt_deployment(deployment) - adapted_deployment["pr_count"] = len(prs) - - adapted_deployments.append(adapted_deployment) - - repo_id_deployments_map[repo_id] = adapted_deployments - - return { - "deployments_map": repo_id_deployments_map, - "repos_map": { - str(repo.id): { - "id": str(repo.id), - "name": repo.name, - "language": repo.language, - "default_branch": repo.default_branch, - "parent": repo.org_name, - } - for repo in org_repos - }, - } - - -@app.route("/deployments//prs", methods={"GET"}) -def get_prs_included_in_deployment(deployment_id: str): - pr_analytics_service = get_pr_analytics_service() - deployment_type: DeploymentType - - ( - deployment_type, - entity_id, - ) = DeploymentsFactoryService.get_deployment_type_and_entity_id_from_deployment_id( - deployment_id - ) - - deployments_service: DeploymentsFactoryService = get_deployments_factory( - deployment_type - ) - deployment: Deployment = deployments_service.get_deployment_by_entity_id(entity_id) - if not deployment: - raise NotFound(f"Deployment not found for id {deployment_id}") - - repo: OrgRepo = pr_analytics_service.get_repo_by_id(deployment.repo_id) - - prs: List[ - PullRequest - ] = deployments_service.get_pull_requests_related_to_deployment(deployment) - repo_id_map = {repo.id: repo} - - return get_non_paginated_pr_response( - prs=prs, repo_id_map=repo_id_map, total_count=len(prs) - ) - - -@app.route("/teams//deployment_frequency", methods={"GET"}) -@queryschema( - Schema( - { - Required("from_time"): All(str, Coerce(datetime.fromisoformat)), - Required("to_time"): All(str, Coerce(datetime.fromisoformat)), - Optional("pr_filter"): All(str, Coerce(json.loads)), - Optional("workflow_filter"): All(str, Coerce(coerce_workflow_filter)), - } - ), -) -def get_team_deployment_frequency( - team_id: str, - from_time: datetime, - to_time: datetime, - pr_filter: Dict = None, - workflow_filter: WorkflowFilter = None, -): - - query_validator = get_query_validator() - interval = query_validator.interval_validator(from_time, to_time) - query_validator.team_validator(team_id) - - pr_filter: PRFilter = apply_pr_filter( - pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] - ) - - deployments_analytics_service = get_deployment_analytics_service() - - team_deployment_frequency_metrics: DeploymentFrequencyMetrics = ( - deployments_analytics_service.get_team_deployment_frequency_metrics( - team_id, interval, pr_filter, workflow_filter - ) - ) - - return adapt_deployment_frequency_metrics(team_deployment_frequency_metrics) - - -@app.route("/teams//deployment_frequency/trends", methods={"GET"}) -@queryschema( - Schema( - { - Required("from_time"): All(str, Coerce(datetime.fromisoformat)), - Required("to_time"): All(str, Coerce(datetime.fromisoformat)), - Optional("pr_filter"): All(str, Coerce(json.loads)), - Optional("workflow_filter"): All(str, Coerce(coerce_workflow_filter)), - } - ), -) -def get_team_deployment_frequency_trends( - team_id: str, - from_time: datetime, - to_time: datetime, - pr_filter: Dict = None, - workflow_filter: WorkflowFilter = None, -): - - query_validator = get_query_validator() - interval = query_validator.interval_validator(from_time, to_time) - query_validator.team_validator(team_id) - - pr_filter: PRFilter = apply_pr_filter( - pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] - ) - - deployments_analytics_service = get_deployment_analytics_service() - - week_to_deployments_count_map: Dict[ - datetime, int - ] = deployments_analytics_service.get_weekly_deployment_frequency_trends( - team_id, interval, pr_filter, workflow_filter - ) - - return { - week.isoformat(): {"count": deployment_count} - for week, deployment_count in week_to_deployments_count_map.items() - } diff --git a/apiserver/dora/api/hello.py b/apiserver/dora/api/hello.py deleted file mode 100644 index 11504aee9..000000000 --- a/apiserver/dora/api/hello.py +++ /dev/null @@ -1,9 +0,0 @@ -from flask import Blueprint - -app = Blueprint("hello", __name__) - - -@app.route("/", methods=["GET"]) -def hello_world(): - - return {"message": "hello world"} diff --git a/apiserver/dora/api/incidents.py b/apiserver/dora/api/incidents.py deleted file mode 100644 index 4852b69f8..000000000 --- a/apiserver/dora/api/incidents.py +++ /dev/null @@ -1,250 +0,0 @@ -import json -from typing import Dict, List - -from datetime import datetime - -from flask import Blueprint -from voluptuous import Required, Schema, Coerce, All, Optional -from dora.service.code.pr_filter import apply_pr_filter -from dora.store.models.code.filter import PRFilter -from dora.store.models.settings import SettingType, EntityType -from dora.service.incidents.models.mean_time_to_recovery import ChangeFailureRateMetrics -from dora.service.deployments.deployment_service import ( - get_deployments_service, -) -from dora.service.deployments.models.models import Deployment -from dora.store.models.code.workflows.filter import WorkflowFilter -from dora.utils.time import Interval -from dora.service.incidents.incidents import get_incident_service -from dora.api.resources.incident_resources import ( - adapt_change_failure_rate, - adapt_deployments_with_related_incidents, - adapt_incident, - adapt_mean_time_to_recovery_metrics, -) -from dora.store.models.incidents import Incident - -from dora.api.request_utils import coerce_workflow_filter, queryschema -from dora.service.query_validator import get_query_validator - -app = Blueprint("incidents", __name__) - - -@app.route("/teams//resolved_incidents", methods={"GET"}) -@queryschema( - Schema( - { - Required("from_time"): All(str, Coerce(datetime.fromisoformat)), - Required("to_time"): All(str, Coerce(datetime.fromisoformat)), - } - ), -) -def get_resolved_incidents(team_id: str, from_time: datetime, to_time: datetime): - - query_validator = get_query_validator() - interval = query_validator.interval_validator(from_time, to_time) - query_validator.team_validator(team_id) - - incident_service = get_incident_service() - - resolved_incidents: List[Incident] = incident_service.get_resolved_team_incidents( - team_id, interval - ) - - # ToDo: Generate a user map - - return [adapt_incident(incident) for incident in resolved_incidents] - - -@app.route("/teams//deployments_with_related_incidents", methods=["GET"]) -@queryschema( - Schema( - { - Required("from_time"): All(str, Coerce(datetime.fromisoformat)), - Required("to_time"): All(str, Coerce(datetime.fromisoformat)), - Optional("pr_filter"): All(str, Coerce(json.loads)), - Optional("workflow_filter"): All(str, Coerce(coerce_workflow_filter)), - } - ), -) -def get_deployments_with_related_incidents( - team_id: str, - from_time: datetime, - to_time: datetime, - pr_filter: dict = None, - workflow_filter: WorkflowFilter = None, -): - query_validator = get_query_validator() - interval = Interval(from_time, to_time) - query_validator.team_validator(team_id) - - pr_filter: PRFilter = apply_pr_filter( - pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] - ) - - deployments: List[ - Deployment - ] = get_deployments_service().get_team_all_deployments_in_interval( - team_id, interval, pr_filter, workflow_filter - ) - - incident_service = get_incident_service() - - incidents: List[Incident] = incident_service.get_team_incidents(team_id, interval) - - deployment_incidents_map: Dict[ - Deployment, List[Incident] - ] = incident_service.get_deployment_incidents_map(deployments, incidents) - - return list( - map( - lambda deployment: adapt_deployments_with_related_incidents( - deployment, deployment_incidents_map - ), - deployments, - ) - ) - - -@app.route("/teams//mean_time_to_recovery", methods=["GET"]) -@queryschema( - Schema( - { - Required("from_time"): All(str, Coerce(datetime.fromisoformat)), - Required("to_time"): All(str, Coerce(datetime.fromisoformat)), - } - ), -) -def get_team_mttr(team_id: str, from_time: datetime, to_time: datetime): - query_validator = get_query_validator() - interval = query_validator.interval_validator(from_time, to_time) - query_validator.team_validator(team_id) - - incident_service = get_incident_service() - - team_mean_time_to_recovery_metrics = ( - incident_service.get_team_mean_time_to_recovery(team_id, interval) - ) - - return adapt_mean_time_to_recovery_metrics(team_mean_time_to_recovery_metrics) - - -@app.route("/teams//mean_time_to_recovery/trends", methods=["GET"]) -@queryschema( - Schema( - { - Required("from_time"): All(str, Coerce(datetime.fromisoformat)), - Required("to_time"): All(str, Coerce(datetime.fromisoformat)), - } - ), -) -def get_team_mttr_trends(team_id: str, from_time: datetime, to_time: datetime): - query_validator = get_query_validator() - interval = query_validator.interval_validator(from_time, to_time) - query_validator.team_validator(team_id) - - incident_service = get_incident_service() - - weekly_mean_time_to_recovery_metrics = ( - incident_service.get_team_mean_time_to_recovery_trends(team_id, interval) - ) - - return { - week.isoformat(): adapt_mean_time_to_recovery_metrics( - mean_time_to_recovery_metrics - ) - for week, mean_time_to_recovery_metrics in weekly_mean_time_to_recovery_metrics.items() - } - - -@app.route("/teams//change_failure_rate", methods=["GET"]) -@queryschema( - Schema( - { - Required("from_time"): All(str, Coerce(datetime.fromisoformat)), - Required("to_time"): All(str, Coerce(datetime.fromisoformat)), - Optional("pr_filter"): All(str, Coerce(json.loads)), - Optional("workflow_filter"): All(str, Coerce(coerce_workflow_filter)), - } - ), -) -def get_team_cfr( - team_id: str, - from_time: datetime, - to_time: datetime, - pr_filter: dict = None, - workflow_filter: WorkflowFilter = None, -): - - query_validator = get_query_validator() - interval = Interval(from_time, to_time) - query_validator.team_validator(team_id) - - pr_filter: PRFilter = apply_pr_filter( - pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] - ) - - deployments: List[ - Deployment - ] = get_deployments_service().get_team_all_deployments_in_interval( - team_id, interval, pr_filter, workflow_filter - ) - - incident_service = get_incident_service() - - incidents: List[Incident] = incident_service.get_team_incidents(team_id, interval) - - team_change_failure_rate: ChangeFailureRateMetrics = ( - incident_service.get_change_failure_rate_metrics(deployments, incidents) - ) - - return adapt_change_failure_rate(team_change_failure_rate) - - -@app.route("/teams//change_failure_rate/trends", methods=["GET"]) -@queryschema( - Schema( - { - Required("from_time"): All(str, Coerce(datetime.fromisoformat)), - Required("to_time"): All(str, Coerce(datetime.fromisoformat)), - Optional("pr_filter"): All(str, Coerce(json.loads)), - Optional("workflow_filter"): All(str, Coerce(coerce_workflow_filter)), - } - ), -) -def get_team_cfr_trends( - team_id: str, - from_time: datetime, - to_time: datetime, - pr_filter: dict = None, - workflow_filter: WorkflowFilter = None, -): - - query_validator = get_query_validator() - interval = Interval(from_time, to_time) - query_validator.team_validator(team_id) - - pr_filter: PRFilter = apply_pr_filter( - pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] - ) - - deployments: List[ - Deployment - ] = get_deployments_service().get_team_all_deployments_in_interval( - team_id, interval, pr_filter, workflow_filter - ) - - incident_service = get_incident_service() - - incidents: List[Incident] = incident_service.get_team_incidents(team_id, interval) - - team_weekly_change_failure_rate: Dict[ - datetime, ChangeFailureRateMetrics - ] = incident_service.get_weekly_change_failure_rate( - interval, deployments, incidents - ) - - return { - week.isoformat(): adapt_change_failure_rate(change_failure_rate) - for week, change_failure_rate in team_weekly_change_failure_rate.items() - } diff --git a/apiserver/dora/api/integrations.py b/apiserver/dora/api/integrations.py deleted file mode 100644 index 1d6a7db08..000000000 --- a/apiserver/dora/api/integrations.py +++ /dev/null @@ -1,95 +0,0 @@ -from flask import Blueprint -from voluptuous import Schema, Optional, Coerce, Range, All, Required - -from dora.api.request_utils import queryschema -from dora.service.external_integrations_service import get_external_integrations_service -from dora.service.query_validator import get_query_validator -from dora.store.models import UserIdentityProvider -from dora.utils.github import github_org_data_multi_thread_worker - -app = Blueprint("integrations", __name__) - -STATUS_TOO_MANY_REQUESTS = 429 - - -@app.route("/orgs//integrations/github/orgs", methods={"GET"}) -def get_github_orgs(org_id: str): - query_validator = get_query_validator() - query_validator.org_validator(org_id) - - external_integrations_service = get_external_integrations_service( - org_id, UserIdentityProvider.GITHUB - ) - try: - orgs = external_integrations_service.get_github_organizations() - except Exception as e: - return e, STATUS_TOO_MANY_REQUESTS - org_data_map = github_org_data_multi_thread_worker(orgs) - - return { - "orgs": [ - { - "login": o.login, - "avatar_url": o.avatar_url, - "web_url": o.html_url, - "repos": org_data_map.get(o.name, {}).get("repos", []), - "members": [], - } - for o in orgs - ] - } - - -@app.route("/orgs//integrations/github/orgs//repos", methods={"GET"}) -@queryschema( - Schema( - { - Optional("page_size", default="30"): All( - str, Coerce(int), Range(min=1, max=100) - ), - Optional("page", default="1"): All(str, Coerce(int), Range(min=1)), - } - ), -) -def get_repos(org_id: str, org_login: str, page_size: int, page: int): - query_validator = get_query_validator() - query_validator.org_validator(org_id) - - external_integrations_service = get_external_integrations_service( - org_id, UserIdentityProvider.GITHUB - ) - # GitHub pages start from 0 and Bitbucket pages start from 1. - # Need to be consistent, hence making standard as page starting from 1 - # and passing a decremented value to GitHub - try: - return external_integrations_service.get_github_org_repos( - org_login, page_size, page - 1 - ) - except Exception as e: - return e, STATUS_TOO_MANY_REQUESTS - - -@app.route( - "/orgs//integrations/github///workflows", - methods={"GET"}, -) -def get_prs_for_repo(org_id: str, gh_org_name: str, gh_org_repo_name: str): - query_validator = get_query_validator() - query_validator.org_validator(org_id) - - external_integrations_service = get_external_integrations_service( - org_id, UserIdentityProvider.GITHUB - ) - - workflows_list = external_integrations_service.get_repo_workflows( - gh_org_name, gh_org_repo_name - ) - - return [ - { - "id": github_workflow.id, - "name": github_workflow.name, - "html_url": github_workflow.html_url, - } - for github_workflow in workflows_list - ] diff --git a/apiserver/dora/api/pull_requests.py b/apiserver/dora/api/pull_requests.py deleted file mode 100644 index d1b664818..000000000 --- a/apiserver/dora/api/pull_requests.py +++ /dev/null @@ -1,169 +0,0 @@ -import json -from datetime import datetime - -from flask import Blueprint -from typing import Dict, List - -from voluptuous import Required, Schema, Coerce, All, Optional -from dora.service.code.models.lead_time import LeadTimeMetrics -from dora.service.code.lead_time import get_lead_time_service -from dora.service.code.pr_filter import apply_pr_filter - -from dora.store.models.code import PRFilter -from dora.store.models.core import Team -from dora.service.query_validator import get_query_validator - -from dora.api.request_utils import queryschema -from dora.api.resources.code_resouces import ( - adapt_lead_time_metrics, - adapt_pull_request, - get_non_paginated_pr_response, -) -from dora.store.models.code.pull_requests import PullRequest -from dora.service.pr_analytics import get_pr_analytics_service -from dora.service.settings.models import ExcludedPRsSetting - -from dora.utils.time import Interval - - -from dora.service.settings.configuration_settings import get_settings_service - -from dora.store.models import SettingType, EntityType - - -app = Blueprint("pull_requests", __name__) - - -@app.route("/teams//prs/excluded", methods={"GET"}) -def get_team_excluded_prs(team_id: str): - - settings = get_settings_service().get_settings( - setting_type=SettingType.EXCLUDED_PRS_SETTING, - entity_id=team_id, - entity_type=EntityType.TEAM, - ) - - if not settings: - return [] - - excluded_pr_setting: ExcludedPRsSetting = settings.specific_settings - - excluded_pr_ids = excluded_pr_setting.excluded_pr_ids - - pr_analytics_service = get_pr_analytics_service() - - prs: List[PullRequest] = pr_analytics_service.get_prs_by_ids(excluded_pr_ids) - - return [adapt_pull_request(pr) for pr in prs] - - -@app.route("/teams//lead_time/prs", methods={"GET"}) -@queryschema( - Schema( - { - Required("from_time"): All(str, Coerce(datetime.fromisoformat)), - Required("to_time"): All(str, Coerce(datetime.fromisoformat)), - Optional("pr_filter"): All(str, Coerce(json.loads)), - } - ), -) -def get_lead_time_prs( - team_id: str, - from_time: datetime, - to_time: datetime, - pr_filter: Dict = None, -): - - query_validator = get_query_validator() - - interval: Interval = query_validator.interval_validator(from_time, to_time) - team: Team = query_validator.team_validator(team_id) - - pr_filter: PRFilter = apply_pr_filter( - pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] - ) - - lead_time_service = get_lead_time_service() - pr_analytics = get_pr_analytics_service() - - repos = pr_analytics.get_team_repos(team_id) - - prs = lead_time_service.get_team_lead_time_prs(team, interval, pr_filter) - - repo_id_repo_map = {repo.id: repo for repo in repos} - return get_non_paginated_pr_response(prs, repo_id_repo_map, len(prs)) - - -@app.route("/teams//lead_time", methods={"GET"}) -@queryschema( - Schema( - { - Required("from_time"): All(str, Coerce(datetime.fromisoformat)), - Required("to_time"): All(str, Coerce(datetime.fromisoformat)), - Optional("pr_filter"): All(str, Coerce(json.loads)), - } - ), -) -def get_team_lead_time( - team_id: str, - from_time: datetime, - to_time: datetime, - pr_filter: Dict = None, -): - - query_validator = get_query_validator() - - interval: Interval = query_validator.interval_validator(from_time, to_time) - team: Team = query_validator.team_validator(team_id) - - pr_filter: PRFilter = apply_pr_filter( - pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] - ) - - lead_time_service = get_lead_time_service() - - teams_average_lead_time_metrics = lead_time_service.get_team_lead_time_metrics( - team, interval, pr_filter - ) - - adapted_lead_time_metrics = adapt_lead_time_metrics(teams_average_lead_time_metrics) - - return adapted_lead_time_metrics - - -@app.route("/teams//lead_time/trends", methods={"GET"}) -@queryschema( - Schema( - { - Required("from_time"): All(str, Coerce(datetime.fromisoformat)), - Required("to_time"): All(str, Coerce(datetime.fromisoformat)), - Optional("pr_filter"): All(str, Coerce(json.loads)), - } - ), -) -def get_team_lead_time_trends( - team_id: str, - from_time: datetime, - to_time: datetime, - pr_filter: Dict = None, -): - - query_validator = get_query_validator() - - interval: Interval = query_validator.interval_validator(from_time, to_time) - team: Team = query_validator.team_validator(team_id) - - pr_filter: PRFilter = apply_pr_filter( - pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] - ) - - lead_time_service = get_lead_time_service() - - weekly_lead_time_metrics_avg_map: Dict[ - datetime, LeadTimeMetrics - ] = lead_time_service.get_team_lead_time_metrics_trends(team, interval, pr_filter) - - return { - week.isoformat(): adapt_lead_time_metrics(average_lead_time_metrics) - for week, average_lead_time_metrics in weekly_lead_time_metrics_avg_map.items() - } diff --git a/apiserver/dora/api/request_utils.py b/apiserver/dora/api/request_utils.py deleted file mode 100644 index 7ccb843bd..000000000 --- a/apiserver/dora/api/request_utils.py +++ /dev/null @@ -1,77 +0,0 @@ -from functools import wraps -from uuid import UUID - -from flask import request -from stringcase import snakecase -from voluptuous import Invalid -from werkzeug.exceptions import BadRequest -from dora.store.models.code.workflows import WorkflowFilter - -from dora.service.workflows.workflow_filter import get_workflow_filter_processor - - -def queryschema(schema): - def decorator(f): - @wraps(f) - def new_func(*args, **kwargs): - try: - query_params = request.args.to_dict() - valid_dict = schema(dict(query_params)) - snaked_kwargs = {snakecase(k): v for k, v in valid_dict.items()} - kwargs.update(snaked_kwargs) - except Invalid as e: - message = "Invalid data: %s (path %s)" % ( - str(e.msg), - ".".join([str(k) for k in e.path]), - ) - raise BadRequest(message) - - return f(*args, **kwargs) - - return new_func - - return decorator - - -def uuid_validator(s: str): - UUID(s) - return s - - -def boolean_validator(s: str): - if s.lower() == "true" or s == "1": - return True - elif s.lower() == "false" or s == "0": - return False - else: - raise ValueError("Not a boolean") - - -def dataschema(schema): - def decorator(f): - @wraps(f) - def new_func(*args, **kwargs): - try: - body = request.json or {} - valid_dict = schema(body) - snaked_kwargs = {snakecase(k): v for k, v in valid_dict.items()} - kwargs.update(snaked_kwargs) - except Invalid as e: - message = "Invalid data: %s (path %s)" % ( - str(e.msg), - ".".join([str(k) for k in e.path]), - ) - raise BadRequest(message) - - return f(*args, **kwargs) - - return new_func - - return decorator - - -def coerce_workflow_filter(filter_data: str) -> WorkflowFilter: - workflow_filter_processor = get_workflow_filter_processor() - return workflow_filter_processor.create_workflow_filter_from_json_string( - filter_data - ) diff --git a/apiserver/dora/api/resources/__init__.py b/apiserver/dora/api/resources/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/dora/api/resources/code_resouces.py b/apiserver/dora/api/resources/code_resouces.py deleted file mode 100644 index 9967c6e72..000000000 --- a/apiserver/dora/api/resources/code_resouces.py +++ /dev/null @@ -1,107 +0,0 @@ -from typing import Dict, List -from dora.service.code.models.lead_time import LeadTimeMetrics -from dora.api.resources.core_resources import adapt_user_info -from dora.store.models.code import PullRequest -from dora.store.models.core import Users - - -def adapt_pull_request( - pr: PullRequest, - username_user_map: Dict[str, Users] = None, -) -> Dict[str, any]: - username_user_map = username_user_map or {} - pr_data = { - "id": str(pr.id), - "repo_id": str(pr.repo_id), - "number": pr.number, - "title": pr.title, - "state": pr.state.value, - "author": adapt_user_info(pr.author, username_user_map), - "reviewers": [ - adapt_user_info(r, username_user_map) for r in (pr.reviewers or []) - ], - "url": pr.url, - "base_branch": pr.base_branch, - "head_branch": pr.head_branch, - "created_at": pr.created_at.isoformat(), - "updated_at": pr.updated_at.isoformat(), - "state_changed_at": pr.state_changed_at.isoformat() - if pr.state_changed_at - else None, - "commits": pr.commits, - "additions": pr.additions, - "deletions": pr.deletions, - "changed_files": pr.changed_files, - "comments": pr.comments, - "provider": pr.provider, - "first_commit_to_open": pr.first_commit_to_open, - "first_response_time": pr.first_response_time, - "rework_time": pr.rework_time, - "merge_time": pr.merge_time, - "merge_to_deploy": pr.merge_to_deploy, - "cycle_time": pr.cycle_time, - "lead_time": pr.lead_time, - "rework_cycles": pr.rework_cycles, - } - - return pr_data - - -def get_non_paginated_pr_response( - prs: List[PullRequest], - repo_id_map: dict, - total_count: int, - username_user_map: dict = None, -): - username_user_map = username_user_map or {} - return { - "data": [ - { - "id": str(pr.id), - "number": pr.number, - "title": pr.title, - "state": pr.state.value, - "first_commit_to_open": pr.first_commit_to_open, - "merge_to_deploy": pr.merge_to_deploy, - "first_response_time": pr.first_response_time, - "rework_time": pr.rework_time, - "merge_time": pr.merge_time, - "cycle_time": pr.cycle_time, - "lead_time": pr.lead_time, - "author": adapt_user_info(pr.author, username_user_map), - "reviewers": [ - adapt_user_info(r, username_user_map) for r in (pr.reviewers or []) - ], - "repo_name": repo_id_map[pr.repo_id].name, - "pr_link": pr.url, - "base_branch": pr.base_branch, - "head_branch": pr.head_branch, - "created_at": pr.created_at.isoformat(), - "updated_at": pr.updated_at.isoformat(), - "state_changed_at": pr.state_changed_at.isoformat() - if pr.state_changed_at - else None, - "commits": pr.commits, - "additions": pr.additions, - "deletions": pr.deletions, - "changed_files": pr.changed_files, - "comments": pr.comments, - "provider": pr.provider, - "rework_cycles": pr.rework_cycles, - } - for pr in prs - ], - "total_count": total_count, - } - - -def adapt_lead_time_metrics(lead_time_metric: LeadTimeMetrics) -> Dict[str, any]: - return { - "lead_time": lead_time_metric.lead_time, - "first_commit_to_open": lead_time_metric.first_commit_to_open, - "first_response_time": lead_time_metric.first_response_time, - "rework_time": lead_time_metric.rework_time, - "merge_time": lead_time_metric.merge_time, - "merge_to_deploy": lead_time_metric.merge_to_deploy, - "pr_count": lead_time_metric.pr_count, - } diff --git a/apiserver/dora/api/resources/core_resources.py b/apiserver/dora/api/resources/core_resources.py deleted file mode 100644 index 63330887d..000000000 --- a/apiserver/dora/api/resources/core_resources.py +++ /dev/null @@ -1,33 +0,0 @@ -from typing import Dict -from dora.store.models.core.teams import Team - -from dora.store.models import Users - - -def adapt_user_info( - author: str, - username_user_map: Dict[str, Users] = None, -): - if not username_user_map or author not in username_user_map: - return {"username": author, "linked_user": None} - - return { - "username": author, - "linked_user": { - "id": str(username_user_map[author].id), - "name": username_user_map[author].name, - "email": username_user_map[author].primary_email, - "avatar_url": username_user_map[author].avatar_url, - }, - } - - -def adapt_team(team: Team): - return { - "id": str(team.id), - "org_id": str(team.org_id), - "name": team.name, - "member_ids": [str(member_id) for member_id in team.member_ids], - "created_at": team.created_at.isoformat(), - "updated_at": team.updated_at.isoformat(), - } diff --git a/apiserver/dora/api/resources/deployment_resources.py b/apiserver/dora/api/resources/deployment_resources.py deleted file mode 100644 index efe401f7a..000000000 --- a/apiserver/dora/api/resources/deployment_resources.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Dict -from .core_resources import adapt_user_info -from dora.store.models.core.users import Users - -from dora.service.deployments.models.models import ( - Deployment, - DeploymentFrequencyMetrics, -) - - -def adapt_deployment( - deployment: Deployment, username_user_map: Dict[str, Users] = None -) -> Dict: - return { - "id": str(deployment.id), - "deployment_type": deployment.deployment_type.value, - "repo_id": str(deployment.repo_id), - "entity_id": str(deployment.entity_id), - "provider": deployment.provider, - "event_actor": adapt_user_info(deployment.actor, username_user_map), - "head_branch": deployment.head_branch, - "conducted_at": deployment.conducted_at.isoformat(), - "duration": deployment.duration, - "status": deployment.status.value, - "html_url": deployment.html_url, - "meta": deployment.meta, - } - - -def adapt_deployment_frequency_metrics( - deployment_frequency_metrics: DeploymentFrequencyMetrics, -) -> Dict: - return { - "total_deployments": deployment_frequency_metrics.total_deployments, - "avg_daily_deployment_frequency": deployment_frequency_metrics.daily_deployment_frequency, - "avg_weekly_deployment_frequency": deployment_frequency_metrics.avg_weekly_deployment_frequency, - "avg_monthly_deployment_frequency": deployment_frequency_metrics.avg_monthly_deployment_frequency, - } diff --git a/apiserver/dora/api/resources/incident_resources.py b/apiserver/dora/api/resources/incident_resources.py deleted file mode 100644 index 587ed6042..000000000 --- a/apiserver/dora/api/resources/incident_resources.py +++ /dev/null @@ -1,71 +0,0 @@ -from typing import Dict, List -from dora.service.incidents.models.mean_time_to_recovery import ( - MeanTimeToRecoveryMetrics, - ChangeFailureRateMetrics, -) -from dora.api.resources.deployment_resources import adapt_deployment -from dora.service.deployments.models.models import Deployment -from dora.store.models.incidents import Incident -from dora.api.resources.core_resources import adapt_user_info - - -def adapt_incident( - incident: Incident, - username_user_map: dict = None, -): - return { - "id": str(incident.id), - "title": incident.title, - "key": incident.key, - "incident_number": incident.incident_number, - "provider": incident.provider, - "status": incident.status, - "creation_date": incident.creation_date.isoformat(), - "resolved_date": incident.resolved_date.isoformat() - if incident.resolved_date - else None, - "acknowledged_date": incident.acknowledged_date.isoformat() - if incident.acknowledged_date - else None, - "assigned_to": adapt_user_info(incident.assigned_to, username_user_map), - "assignees": list( - map( - lambda assignee: adapt_user_info(assignee, username_user_map), - incident.assignees or [], - ) - ), - "url": None, # ToDo: Add URL to incidents - "summary": incident.meta.get("summary"), - "incident_type": incident.incident_type.value, - } - - -def adapt_deployments_with_related_incidents( - deployment: Deployment, - deployment_incidents_map: Dict[Deployment, List[Incident]], - username_user_map: dict = None, -): - deployment_response = adapt_deployment(deployment, username_user_map) - incidents = deployment_incidents_map.get(deployment, []) - incident_response = list( - map(lambda incident: adapt_incident(incident, username_user_map), incidents) - ) - deployment_response["incidents"] = incident_response - return deployment_response - - -def adapt_mean_time_to_recovery_metrics( - mean_time_to_recovery: MeanTimeToRecoveryMetrics, -): - return { - "mean_time_to_recovery": mean_time_to_recovery.mean_time_to_recovery, - "incident_count": mean_time_to_recovery.incident_count, - } - - -def adapt_change_failure_rate(change_failure_rate: ChangeFailureRateMetrics): - return { - "change_failure_rate": change_failure_rate.change_failure_rate, - "failed_deployments": change_failure_rate.failed_deployments_count, - "total_deployments": change_failure_rate.total_deployments_count, - } diff --git a/apiserver/dora/api/resources/settings_resource.py b/apiserver/dora/api/resources/settings_resource.py deleted file mode 100644 index a932935bd..000000000 --- a/apiserver/dora/api/resources/settings_resource.py +++ /dev/null @@ -1,61 +0,0 @@ -from dora.service.settings.models import ( - ConfigurationSettings, - IncidentSettings, - ExcludedPRsSetting, - IncidentTypesSetting, - IncidentSourcesSetting, -) -from dora.store.models import EntityType - - -def adapt_configuration_settings_response(config_settings: ConfigurationSettings): - def _add_entity(config_settings: ConfigurationSettings, response): - - if config_settings.entity_type == EntityType.USER: - response["user_id"] = str(config_settings.entity_id) - - if config_settings.entity_type == EntityType.TEAM: - response["team_id"] = str(config_settings.entity_id) - - if config_settings.entity_type == EntityType.ORG: - response["org_id"] = str(config_settings.entity_id) - - return response - - def _add_setting_data(config_settings: ConfigurationSettings, response): - - # Add new if statements to add settings response for new settings - if isinstance(config_settings.specific_settings, IncidentSettings): - response["setting"] = { - "title_includes": config_settings.specific_settings.title_filters - } - if isinstance(config_settings.specific_settings, ExcludedPRsSetting): - response["setting"] = { - "excluded_pr_ids": config_settings.specific_settings.excluded_pr_ids - } - - if isinstance(config_settings.specific_settings, IncidentTypesSetting): - response["setting"] = { - "incident_types": [ - incident_type.value - for incident_type in config_settings.specific_settings.incident_types - ] - } - - if isinstance(config_settings.specific_settings, IncidentSourcesSetting): - response["setting"] = { - "incident_sources": [ - source.value - for source in config_settings.specific_settings.incident_sources - ] - } - - return response - - response = { - "created_at": config_settings.created_at.isoformat(), - "updated_at": config_settings.updated_at.isoformat(), - } - response = _add_entity(config_settings, response) - response = _add_setting_data(config_settings, response) - return response diff --git a/apiserver/dora/api/settings.py b/apiserver/dora/api/settings.py deleted file mode 100644 index 179ef94eb..000000000 --- a/apiserver/dora/api/settings.py +++ /dev/null @@ -1,172 +0,0 @@ -from typing import Dict - -from flask import Blueprint -from voluptuous import Required, Schema, Coerce, All, Optional -from werkzeug.exceptions import BadRequest, NotFound - -from dora.api.request_utils import dataschema, queryschema, uuid_validator -from dora.api.resources.settings_resource import adapt_configuration_settings_response -from dora.service.query_validator import get_query_validator -from dora.service.settings import get_settings_service, settings_type_validator -from dora.store.models import Organization, Users, SettingType, EntityType - -app = Blueprint("settings", __name__) - - -@app.route("/teams//settings", methods={"GET"}) -@queryschema( - Schema( - { - Required("setting_type"): All(str, Coerce(settings_type_validator)), - Optional("setter_id"): All(str, Coerce(uuid_validator)), - } - ), -) -def get_team_settings(team_id: str, setting_type: SettingType, setter_id: str = None): - - query_validator = get_query_validator() - - team = query_validator.team_validator(team_id) - - setter = None - - if setter_id: - setter = query_validator.user_validator(setter_id) - - if setter and str(setter.org_id) != str(team.org_id): - raise BadRequest(f"User {setter_id} does not belong to team {team_id}") - - settings_service = get_settings_service() - settings = settings_service.get_settings( - setting_type=setting_type, - entity_type=EntityType.TEAM, - entity_id=team_id, - ) - - if not settings: - settings = settings_service.save_settings( - setting_type=setting_type, - entity_type=EntityType.TEAM, - entity_id=team_id, - setter=setter, - ) - - return adapt_configuration_settings_response(settings) - - -@app.route("/teams//settings", methods={"PUT"}) -@dataschema( - Schema( - { - Required("setting_type"): All(str, Coerce(settings_type_validator)), - Optional("setter_id"): All(str, Coerce(uuid_validator)), - Required("setting_data"): dict, - } - ), -) -def put_team_settings( - team_id: str, - setting_type: SettingType, - setter_id: str = None, - setting_data: Dict = None, -): - - query_validator = get_query_validator() - - team = query_validator.team_validator(team_id) - - setter = None - - if setter_id: - setter = query_validator.user_validator(setter_id) - - if setter and str(setter.org_id) != str(team.org_id): - raise BadRequest(f"User {setter_id} does not belong to team {team_id}") - - settings_service = get_settings_service() - settings = settings_service.save_settings( - setting_type=setting_type, - entity_type=EntityType.TEAM, - entity_id=team_id, - setter=setter, - setting_data=setting_data, - ) - return adapt_configuration_settings_response(settings) - - -@app.route("/orgs//settings", methods={"GET"}) -@queryschema( - Schema( - { - Required("setting_type"): All(str, Coerce(settings_type_validator)), - Optional("setter_id"): All(str, Coerce(uuid_validator)), - } - ), -) -def get_org_settings(org_id: str, setting_type: SettingType, setter_id: str = None): - - query_validator = get_query_validator() - org: Organization = query_validator.org_validator(org_id) - - setter = None - - if setter_id: - setter = query_validator.user_validator(setter_id) - - if setter and str(setter.org_id) != str(org_id): - raise BadRequest(f"User {setter_id} does not belong to org {org_id}") - - settings_service = get_settings_service() - settings = settings_service.get_settings( - setting_type=setting_type, - entity_type=EntityType.ORG, - entity_id=org_id, - ) - - if not settings: - settings = settings_service.save_settings( - setting_type=setting_type, - entity_type=EntityType.ORG, - entity_id=org_id, - setter=setter, - ) - - return adapt_configuration_settings_response(settings) - - -@app.route("/orgs//settings", methods={"PUT"}) -@dataschema( - Schema( - { - Required("setting_type"): All(str, Coerce(settings_type_validator)), - Optional("setter_id"): All(str, Coerce(uuid_validator)), - Required("setting_data"): dict, - } - ), -) -def put_org_settings( - org_id: str, - setting_type: SettingType, - setter_id: str = None, - setting_data: Dict = None, -): - query_validator = get_query_validator() - org: Organization = query_validator.org_validator(org_id) - - setter = None - - if setter_id: - setter = query_validator.user_validator(setter_id) - - if setter and str(setter.org_id) != str(org_id): - raise BadRequest(f"User {setter_id} does not belong to org {org_id}") - - settings_service = get_settings_service() - settings = settings_service.save_settings( - setting_type=setting_type, - entity_type=EntityType.ORG, - entity_id=org_id, - setter=setter, - setting_data=setting_data, - ) - return adapt_configuration_settings_response(settings) diff --git a/apiserver/dora/api/sync.py b/apiserver/dora/api/sync.py deleted file mode 100644 index ce75d2bd6..000000000 --- a/apiserver/dora/api/sync.py +++ /dev/null @@ -1,12 +0,0 @@ -from flask import Blueprint - -from dora.service.sync_data import trigger_data_sync -from dora.utils.time import time_now - -app = Blueprint("sync", __name__) - - -@app.route("/sync", methods=["POST"]) -def sync(): - trigger_data_sync() - return {"message": "sync started", "time": time_now().isoformat()} diff --git a/apiserver/dora/api/teams.py b/apiserver/dora/api/teams.py deleted file mode 100644 index b2b484dc5..000000000 --- a/apiserver/dora/api/teams.py +++ /dev/null @@ -1,79 +0,0 @@ -from flask import Blueprint -from typing import List -from voluptuous import Required, Schema, Optional -from dora.api.resources.core_resources import adapt_team -from dora.store.models.core.teams import Team -from dora.service.core.teams import get_team_service - -from dora.api.request_utils import dataschema -from dora.service.query_validator import get_query_validator - -app = Blueprint("teams", __name__) - - -@app.route("/team/", methods={"GET"}) -def fetch_team(team_id): - - query_validator = get_query_validator() - team: Team = query_validator.team_validator(team_id) - - return adapt_team(team) - - -@app.route("/team/", methods={"PATCH"}) -@dataschema( - Schema( - { - Optional("name"): str, - Optional("member_ids"): list, - } - ), -) -def update_team_patch(team_id: str, name: str = None, member_ids: List[str] = None): - - query_validator = get_query_validator() - team: Team = query_validator.team_validator(team_id) - - if member_ids: - query_validator.users_validator(member_ids) - - team_service = get_team_service() - - team: Team = team_service.update_team(team_id, name, member_ids) - - return adapt_team(team) - - -@app.route("/org//team", methods={"POST"}) -@dataschema( - Schema( - { - Required("name"): str, - Required("member_ids"): list, - } - ), -) -def create_team(org_id: str, name: str, member_ids: List[str]): - - query_validator = get_query_validator() - query_validator.org_validator(org_id) - query_validator.users_validator(member_ids) - - team_service = get_team_service() - - team: Team = team_service.create_team(org_id, name, member_ids) - - return adapt_team(team) - - -@app.route("/team/", methods={"DELETE"}) -def delete_team(team_id: str): - - query_validator = get_query_validator() - team: Team = query_validator.team_validator(team_id) - - team_service = get_team_service() - - team = team_service.delete_team(team_id) - - return adapt_team(team) diff --git a/apiserver/dora/config/config.ini b/apiserver/dora/config/config.ini deleted file mode 100644 index 75da19b09..000000000 --- a/apiserver/dora/config/config.ini +++ /dev/null @@ -1,3 +0,0 @@ -[KEYS] -SECRET_PRIVATE_KEY = SECRET_PRIVATE_KEY -SECRET_PUBLIC_KEY = SECRET_PUBLIC_KEY \ No newline at end of file diff --git a/apiserver/dora/exapi/__init__.py b/apiserver/dora/exapi/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/dora/exapi/git_incidents.py b/apiserver/dora/exapi/git_incidents.py deleted file mode 100644 index 8eb0fb501..000000000 --- a/apiserver/dora/exapi/git_incidents.py +++ /dev/null @@ -1,74 +0,0 @@ -from datetime import datetime -from typing import List - -from dora.exapi.models.git_incidents import RevertPRMap -from dora.service.settings import SettingsService, get_settings_service -from dora.store.models import SettingType, EntityType -from dora.store.models.code import PullRequest, PullRequestRevertPRMapping -from dora.store.models.incidents import IncidentSource -from dora.store.repos.code import CodeRepoService - - -class GitIncidentsAPIService: - def __init__( - self, code_repo_service: CodeRepoService, settings_service: SettingsService - ): - self.code_repo_service = code_repo_service - self.settings_service = settings_service - - def is_sync_enabled(self, org_id: str): - setting = self.settings_service.get_settings( - setting_type=SettingType.INCIDENT_SOURCES_SETTING, - entity_type=EntityType.ORG, - entity_id=org_id, - ) - if setting: - incident_sources_setting = setting.specific_settings - else: - incident_sources_setting = self.settings_service.get_default_setting( - SettingType.INCIDENT_SOURCES_SETTING - ) - incident_sources = incident_sources_setting.incident_sources - return IncidentSource.GIT_REPO in incident_sources - - def get_org_repos(self, org_id: str): - return self.code_repo_service.get_active_org_repos(org_id) - - def get_org_repo(self, repo_id: str): - return self.code_repo_service.get_repo_by_id(repo_id) - - def get_repo_revert_prs_in_interval( - self, repo_id: str, from_time: datetime, to_time: datetime - ) -> List[RevertPRMap]: - revert_pr_mappings: List[ - PullRequestRevertPRMapping - ] = self.code_repo_service.get_repo_revert_prs_mappings_updated_in_interval( - repo_id, from_time, to_time - ) - - revert_pr_ids = [str(pr.pr_id) for pr in revert_pr_mappings] - original_pr_ids = [str(pr.reverted_pr) for pr in revert_pr_mappings] - prs: List[PullRequest] = self.code_repo_service.get_prs_by_ids( - revert_pr_ids + original_pr_ids - ) - id_to_pr_map = {str(pr.id): pr for pr in prs} - - revert_prs: List[RevertPRMap] = [] - for mapping in revert_pr_mappings: - revert_pr = id_to_pr_map.get(str(mapping.pr_id)) - original_pr = id_to_pr_map.get(str(mapping.reverted_pr)) - if revert_pr and original_pr: - revert_prs.append( - RevertPRMap( - revert_pr=revert_pr, - original_pr=original_pr, - created_at=mapping.created_at, - updated_at=mapping.updated_at, - ) - ) - - return revert_prs - - -def get_git_incidents_api_service(): - return GitIncidentsAPIService(CodeRepoService(), get_settings_service()) diff --git a/apiserver/dora/exapi/github.py b/apiserver/dora/exapi/github.py deleted file mode 100644 index 6ee5c0d25..000000000 --- a/apiserver/dora/exapi/github.py +++ /dev/null @@ -1,254 +0,0 @@ -import contextlib -from datetime import datetime -from http import HTTPStatus -from typing import Optional, Dict, Tuple, List - -import requests -from github import Github, UnknownObjectException -from github.GithubException import RateLimitExceededException -from github.Organization import Organization as GithubOrganization -from github.PaginatedList import PaginatedList as GithubPaginatedList -from github.PullRequest import PullRequest as GithubPullRequest -from github.Repository import Repository as GithubRepository - -from dora.exapi.models.github import GitHubContributor -from dora.utils.log import LOG - -PAGE_SIZE = 100 - - -class GithubRateLimitExceeded(Exception): - pass - - -class GithubApiService: - def __init__(self, access_token: str): - self._token = access_token - self._g = Github(self._token, per_page=PAGE_SIZE) - self.base_url = "https://api.github.com" - self.headers = {"Authorization": f"Bearer {self._token}"} - - @contextlib.contextmanager - def temp_config(self, per_page: int = 30): - self._g.per_page = per_page - yield - self._g.per_page = PAGE_SIZE - - def check_pat(self) -> bool: - """ - Checks if PAT is Valid - :returns: - :raises HTTPError: If the request fails and status code is not 200 - """ - url = f"{self.base_url}/user" - response = requests.get(url, headers=self.headers) - return response.status_code == 200 - - def get_org_list(self) -> [GithubOrganization]: - try: - orgs = list(self._g.get_user().get_orgs()) - except RateLimitExceededException: - raise GithubRateLimitExceeded("GITHUB_API_LIMIT_EXCEEDED") - - return orgs - - def get_repos( - self, org_login: str, per_page: int = 30, page: int = 0 - ) -> [GithubRepository]: - with self.temp_config( - per_page=per_page - ): # This works on assumption of single thread, else make thread local - o = self._g.get_organization(org_login) - repos = o.get_repos().get_page(page) - return repos - - def get_repos_raw( - self, org_login: str, per_page: int = 30, page: int = 0 - ) -> [Dict]: - try: - repos = self.get_repos(org_login, per_page, page) - except RateLimitExceededException: - raise GithubRateLimitExceeded("GITHUB_API_LIMIT_EXCEEDED") - - return [repo.__dict__["_rawData"] for repo in repos] - - def get_repo(self, org_login: str, repo_name: str) -> Optional[GithubRepository]: - try: - return self._g.get_repo(f"{org_login}/{repo_name}") - except UnknownObjectException: - return None - - def get_repo_contributors(self, github_repo: GithubRepository) -> [Tuple[str, int]]: - contributors = list(github_repo.get_contributors()) - return [(u.login, u.contributions) for u in contributors] - - def get_pull_requests( - self, repo: GithubRepository, state="all", sort="updated", direction="desc" - ) -> GithubPaginatedList: - return repo.get_pulls(state=state, sort=sort, direction=direction) - - def get_raw_prs(self, prs: [GithubPullRequest]): - return [pr.__dict__["_rawData"] for pr in prs] - - def get_pull_request( - self, github_repo: GithubRepository, number: int - ) -> GithubPullRequest: - return github_repo.get_pull(number=number) - - def get_pr_commits(self, pr: GithubPullRequest): - return pr.get_commits() - - def get_pr_reviews(self, pr: GithubPullRequest) -> GithubPaginatedList: - return pr.get_reviews() - - def get_contributors( - self, org_login: str, repo_name: str - ) -> List[GitHubContributor]: - - gh_contributors_list = [] - page = 1 - - def _get_contributor_data_from_dict(contributor) -> GitHubContributor: - return GitHubContributor( - login=contributor["login"], - id=contributor["id"], - node_id=contributor["node_id"], - avatar_url=contributor["avatar_url"], - contributions=contributor["contributions"], - events_url=contributor["events_url"], - followers_url=contributor["followers_url"], - following_url=contributor["following_url"], - site_admin=contributor["site_admin"], - gists_url=contributor["gists_url"], - gravatar_id=contributor["gravatar_id"], - html_url=contributor["html_url"], - organizations_url=contributor["organizations_url"], - received_events_url=contributor["received_events_url"], - repos_url=contributor["repos_url"], - starred_url=contributor["starred_url"], - type=contributor["type"], - subscriptions_url=contributor["subscriptions_url"], - url=contributor["url"], - ) - - def _fetch_contributors(page: int = 0): - github_url = f"{self.base_url}/repos/{org_login}/{repo_name}/contributors" - query_params = dict(per_page=PAGE_SIZE, page=page) - response = requests.get( - github_url, headers=self.headers, params=query_params - ) - assert response.status_code == HTTPStatus.OK - return response.json() - - data = _fetch_contributors(page=page) - while data: - gh_contributors_list += data - if len(data) < PAGE_SIZE: - break - - page += 1 - data = _fetch_contributors(page=page) - - contributors: List[GitHubContributor] = [ - _get_contributor_data_from_dict(contributor) - for contributor in gh_contributors_list - ] - return contributors - - def get_org_members(self, org_login: str) -> List[GitHubContributor]: - - gh_org_member_list = [] - page = 1 - - def _get_contributor_data_from_dict(contributor) -> GitHubContributor: - return GitHubContributor( - login=contributor["login"], - id=contributor["id"], - node_id=contributor["node_id"], - avatar_url=contributor["avatar_url"], - events_url=contributor["events_url"], - followers_url=contributor["followers_url"], - following_url=contributor["following_url"], - site_admin=contributor["site_admin"], - gists_url=contributor["gists_url"], - gravatar_id=contributor["gravatar_id"], - html_url=contributor["html_url"], - organizations_url=contributor["organizations_url"], - received_events_url=contributor["received_events_url"], - repos_url=contributor["repos_url"], - starred_url=contributor["starred_url"], - type=contributor["type"], - subscriptions_url=contributor["subscriptions_url"], - url=contributor["url"], - contributions=0, - ) - - def _fetch_members(page: int = 0): - github_url = f"{self.base_url}/orgs/{org_login}/members" - query_params = dict(per_page=PAGE_SIZE, page=page) - response = requests.get( - github_url, headers=self.headers, params=query_params - ) - assert response.status_code == HTTPStatus.OK - return response.json() - - data = _fetch_members(page=page) - while data: - gh_org_member_list += data - if len(data) < PAGE_SIZE: - break - - page += 1 - data = _fetch_members(page=page) - - members: List[GitHubContributor] = [ - _get_contributor_data_from_dict(contributor) - for contributor in gh_org_member_list - ] - return members - - def get_repo_workflows( - self, org_login: str, repo_name: str - ) -> Optional[GithubPaginatedList]: - try: - return self._g.get_repo(f"{org_login}/{repo_name}").get_workflows() - except UnknownObjectException: - return None - - def get_workflow_runs( - self, org_login: str, repo_name: str, workflow_id: str, bookmark: datetime - ): - repo_workflows = [] - page = 1 - - def _fetch_workflow_runs(page: int = 1): - github_url = f"{self.base_url}/repos/{org_login}/{repo_name}/actions/workflows/{workflow_id}/runs" - query_params = dict( - per_page=PAGE_SIZE, - page=page, - created=f"created:>={bookmark.isoformat()}", - ) - response = requests.get( - github_url, headers=self.headers, params=query_params - ) - - if response.status_code == HTTPStatus.NOT_FOUND: - LOG.error( - f"[GitHub Sync Repo Workflow Worker] Workflow {workflow_id} Not found " - f"for repo {org_login}/{repo_name}" - ) - return {} - - assert response.status_code == HTTPStatus.OK - return response.json() - - data = _fetch_workflow_runs(page=page) - while data and data.get("workflow_runs"): - curr_workflow_repos = data.get("workflow_runs") - repo_workflows += curr_workflow_repos - if len(curr_workflow_repos) == 0: - break - - page += 1 - data = _fetch_workflow_runs(page=page) - return repo_workflows diff --git a/apiserver/dora/exapi/models/__init__.py b/apiserver/dora/exapi/models/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/dora/exapi/models/git_incidents.py b/apiserver/dora/exapi/models/git_incidents.py deleted file mode 100644 index 4931741d2..000000000 --- a/apiserver/dora/exapi/models/git_incidents.py +++ /dev/null @@ -1,12 +0,0 @@ -from dataclasses import dataclass -from datetime import datetime - -from dora.store.models.code import PullRequest - - -@dataclass -class RevertPRMap: - revert_pr: PullRequest - original_pr: PullRequest - created_at: datetime - updated_at: datetime diff --git a/apiserver/dora/exapi/models/github.py b/apiserver/dora/exapi/models/github.py deleted file mode 100644 index ee0057f45..000000000 --- a/apiserver/dora/exapi/models/github.py +++ /dev/null @@ -1,45 +0,0 @@ -from dataclasses import dataclass - - -@dataclass -class GitHubBaseUser: - login: str = "" - id: int = 0 - node_id: str = "" - avatar_url: str = "" - gravatar_id: str = "" - url: str = "" - html_url: str = "" - followers_url: str = "" - following_url: str = "" - gists_url: str = "" - starred_url: str = "" - subscriptions_url: str = "" - organizations_url: str = "" - repos_url: str = "" - events_url: str = "" - received_events_url: str = "" - type: str = "User" - site_admin: bool = False - contributions: int = 0 - - def __hash__(self): - return hash(self.id) - - def __eq__(self, other): - if isinstance(other, GitHubBaseUser): - return self.id == other.id - return False - - -@dataclass -class GitHubContributor(GitHubBaseUser): - contributions: int = 0 - - def __hash__(self): - return hash(self.id) - - def __eq__(self, other): - if isinstance(other, GitHubContributor): - return self.id == other.id - return False diff --git a/apiserver/dora/service/__init__.py b/apiserver/dora/service/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/dora/service/code/__init__.py b/apiserver/dora/service/code/__init__.py deleted file mode 100644 index 5166d910c..000000000 --- a/apiserver/dora/service/code/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .sync import sync_code_repos -from .integration import get_code_integration_service -from .pr_filter import apply_pr_filter diff --git a/apiserver/dora/service/code/integration.py b/apiserver/dora/service/code/integration.py deleted file mode 100644 index 7e195a05d..000000000 --- a/apiserver/dora/service/code/integration.py +++ /dev/null @@ -1,27 +0,0 @@ -from typing import List - -from dora.store.models import UserIdentityProvider, Integration -from dora.store.repos.core import CoreRepoService - -CODE_INTEGRATION_BUCKET = [ - UserIdentityProvider.GITHUB.value, -] - - -class CodeIntegrationService: - def __init__(self, core_repo_service: CoreRepoService): - self.core_repo_service = core_repo_service - - def get_org_providers(self, org_id: str) -> List[str]: - integrations: List[ - Integration - ] = self.core_repo_service.get_org_integrations_for_names( - org_id, CODE_INTEGRATION_BUCKET - ) - if not integrations: - return [] - return [integration.name for integration in integrations] - - -def get_code_integration_service(): - return CodeIntegrationService(core_repo_service=CoreRepoService()) diff --git a/apiserver/dora/service/code/lead_time.py b/apiserver/dora/service/code/lead_time.py deleted file mode 100644 index 30fe32afc..000000000 --- a/apiserver/dora/service/code/lead_time.py +++ /dev/null @@ -1,264 +0,0 @@ -from typing import Dict, List -from datetime import datetime -from dora.service.code.models.lead_time import LeadTimeMetrics -from dora.store.models.code.repository import TeamRepos - -from dora.store.models.code import PRFilter, PullRequest -from dora.store.repos.code import CodeRepoService -from dora.store.models.core import Team - -from dora.service.deployments.deployment_service import ( - DeploymentsService, - get_deployments_service, -) - -from dora.utils.time import ( - Interval, - fill_missing_week_buckets, - generate_expanded_buckets, -) - - -class LeadTimeService: - def __init__( - self, - code_repo_service: CodeRepoService, - deployments_service: DeploymentsService, - ): - self._code_repo_service = code_repo_service - self._deployments_service = deployments_service - - def get_team_lead_time_metrics( - self, - team: Team, - interval: Interval, - pr_filter: Dict[str, PRFilter] = None, - ) -> LeadTimeMetrics: - - team_repos = self._code_repo_service.get_active_team_repos_by_team_id(team.id) - - return self._get_weighted_avg_lead_time_metrics( - self._get_team_repos_lead_time_metrics(team_repos, interval, pr_filter) - ) - - def get_team_lead_time_metrics_trends( - self, - team: Team, - interval: Interval, - pr_filter: Dict[str, PRFilter] = None, - ) -> Dict[datetime, LeadTimeMetrics]: - - team_repos = self._code_repo_service.get_active_team_repos_by_team_id(team.id) - - lead_time_metrics: List[LeadTimeMetrics] = list( - set(self._get_team_repos_lead_time_metrics(team_repos, interval, pr_filter)) - ) - - weekly_lead_time_metrics_map: Dict[ - datetime, List[LeadTimeMetrics] - ] = generate_expanded_buckets( - lead_time_metrics, interval, "merged_at", "weekly" - ) - - weekly_lead_time_metrics_avg_map: Dict[ - datetime, LeadTimeMetrics - ] = self.get_avg_lead_time_metrics_from_map(weekly_lead_time_metrics_map) - - weekly_lead_time_metrics_avg_map = fill_missing_week_buckets( - weekly_lead_time_metrics_avg_map, interval, LeadTimeMetrics - ) - - return weekly_lead_time_metrics_avg_map - - def get_avg_lead_time_metrics_from_map( - self, map_lead_time_metrics: Dict[datetime, List[LeadTimeMetrics]] - ) -> Dict[datetime, LeadTimeMetrics]: - map_avg_lead_time_metrics = {} - for key, lead_time_metrics in map_lead_time_metrics.items(): - map_avg_lead_time_metrics[key] = self._get_weighted_avg_lead_time_metrics( - lead_time_metrics - ) - return map_avg_lead_time_metrics - - def get_team_lead_time_prs( - self, - team: Team, - interval: Interval, - pr_filter: PRFilter = None, - ) -> List[PullRequest]: - - team_repos = self._code_repo_service.get_active_team_repos_by_team_id(team.id) - - ( - team_repos_using_workflow_deployments, - team_repos_using_pr_deployments, - ) = self._deployments_service.get_filtered_team_repos_by_deployment_config( - team_repos - ) - - lead_time_prs_using_workflow = ( - self._get_lead_time_prs_for_repos_using_workflow_deployments( - team_repos_using_workflow_deployments, interval, pr_filter - ) - ) - - lead_time_prs_using_pr = self._get_lead_time_prs_for_repos_using_pr_deployments( - team_repos_using_pr_deployments, interval, pr_filter - ) - - return list(set(lead_time_prs_using_workflow + lead_time_prs_using_pr)) - - def _get_team_repos_lead_time_metrics( - self, - team_repos: TeamRepos, - interval: Interval, - pr_filter: Dict[str, PRFilter] = None, - ) -> List[LeadTimeMetrics]: - - ( - team_repos_using_workflow_deployments, - team_repos_using_pr_deployments, - ) = self._deployments_service.get_filtered_team_repos_by_deployment_config( - team_repos - ) - - lead_time_metrics_using_workflow = ( - self._get_lead_time_metrics_for_repos_using_workflow_deployments( - team_repos_using_workflow_deployments, interval, pr_filter - ) - ) - - lead_time_metrics_using_pr = ( - self._get_lead_time_metrics_for_repos_using_pr_deployments( - team_repos_using_pr_deployments, interval, pr_filter - ) - ) - - return lead_time_metrics_using_workflow + lead_time_metrics_using_pr - - def _get_lead_time_metrics_for_repos_using_workflow_deployments( - self, - team_repos: List[TeamRepos], - interval: Interval, - pr_filter: PRFilter = None, - ) -> List[LeadTimeMetrics]: - - prs = self._get_lead_time_prs_for_repos_using_workflow_deployments( - team_repos, interval, pr_filter - ) - - pr_lead_time_metrics = [self._get_lead_time_metrics_for_pr(pr) for pr in prs] - - return pr_lead_time_metrics - - def _get_lead_time_metrics_for_repos_using_pr_deployments( - self, - team_repos: List[TeamRepos], - interval: Interval, - pr_filter: PRFilter = None, - ) -> Dict[TeamRepos, List[LeadTimeMetrics]]: - - prs = self._get_lead_time_prs_for_repos_using_pr_deployments( - team_repos, interval, pr_filter - ) - - pr_lead_time_metrics = [self._get_lead_time_metrics_for_pr(pr) for pr in prs] - - for prm in pr_lead_time_metrics: - prm.merge_to_deploy = 0 - - return pr_lead_time_metrics - - def _get_lead_time_prs_for_repos_using_workflow_deployments( - self, - team_repos: List[TeamRepos], - interval: Interval, - pr_filter: PRFilter = None, - ) -> List[PullRequest]: - - team_repos_with_workflow_deployments_configured: List[ - TeamRepos - ] = self._deployments_service.get_filtered_team_repos_with_workflow_configured_deployments( - team_repos - ) - - repo_ids = [ - tr.org_repo_id for tr in team_repos_with_workflow_deployments_configured - ] - - prs = self._code_repo_service.get_prs_merged_in_interval( - repo_ids, - interval, - pr_filter, - has_non_null_mtd=True, - ) - - return prs - - def _get_lead_time_prs_for_repos_using_pr_deployments( - self, - team_repos: List[TeamRepos], - interval: Interval, - pr_filter: PRFilter = None, - ) -> List[PullRequest]: - repo_ids = [tr.org_repo_id for tr in team_repos] - - prs = self._code_repo_service.get_prs_merged_in_interval( - repo_ids, interval, pr_filter - ) - - return prs - - def _get_lead_time_metrics_for_pr(self, pr: PullRequest) -> LeadTimeMetrics: - return LeadTimeMetrics( - first_commit_to_open=pr.first_commit_to_open - if pr.first_commit_to_open is not None and pr.first_commit_to_open > 0 - else 0, - first_response_time=pr.first_response_time if pr.first_response_time else 0, - rework_time=pr.rework_time if pr.rework_time else 0, - merge_time=pr.merge_time if pr.merge_time else 0, - merge_to_deploy=pr.merge_to_deploy if pr.merge_to_deploy else 0, - pr_count=1, - merged_at=pr.state_changed_at, - pr_id=pr.id, - ) - - def _get_weighted_avg_lead_time_metrics( - self, lead_time_metrics: List[LeadTimeMetrics] - ) -> LeadTimeMetrics: - return LeadTimeMetrics( - first_commit_to_open=self._get_avg_time( - lead_time_metrics, "first_commit_to_open" - ), - first_response_time=self._get_avg_time( - lead_time_metrics, "first_response_time" - ), - rework_time=self._get_avg_time(lead_time_metrics, "rework_time"), - merge_time=self._get_avg_time(lead_time_metrics, "merge_time"), - merge_to_deploy=self._get_avg_time(lead_time_metrics, "merge_to_deploy"), - pr_count=sum( - [lead_time_metric.pr_count for lead_time_metric in lead_time_metrics] - ), - ) - - def _get_avg_time( - self, lead_time_metrics: List[LeadTimeMetrics], field: str - ) -> float: - total_pr_count = sum( - [lead_time_metric.pr_count for lead_time_metric in lead_time_metrics] - ) - if total_pr_count == 0: - return 0 - - weighted_sum = sum( - [ - getattr(lead_time_metric, field) * lead_time_metric.pr_count - for lead_time_metric in lead_time_metrics - ] - ) - avg = weighted_sum / total_pr_count - return avg - - -def get_lead_time_service() -> LeadTimeService: - return LeadTimeService(CodeRepoService(), get_deployments_service()) diff --git a/apiserver/dora/service/code/models/lead_time.py b/apiserver/dora/service/code/models/lead_time.py deleted file mode 100644 index cd0f8ea4d..000000000 --- a/apiserver/dora/service/code/models/lead_time.py +++ /dev/null @@ -1,49 +0,0 @@ -from dataclasses import dataclass -from typing import Optional -from datetime import datetime - - -@dataclass -class LeadTimeMetrics: - first_commit_to_open: float = 0 - first_response_time: float = 0 - rework_time: float = 0 - merge_time: float = 0 - merge_to_deploy: float = 0 - pr_count: float = 0 - - merged_at: Optional[datetime] = None - pr_id: Optional[str] = None - - def __eq__(self, other): - if not isinstance(other, LeadTimeMetrics): - raise ValueError( - f"Cannot compare type: LeadTimeMetrics with type: {type(other)}" - ) - if self.pr_id is None: - raise ValueError("PR ID is None") - return self.pr_id == other.pr_id - - def __hash__(self): - if self.pr_id is None: - raise ValueError("PR ID is None") - return hash(self.pr_id) - - @property - def lead_time(self): - return ( - self.first_commit_to_open - + self.first_response_time - + self.rework_time - + self.merge_time - + self.merge_to_deploy - ) - - @property - def cycle_time(self): - return ( - self.first_response_time - + self.rework_time - + self.merge_time - + self.merge_to_deploy - ) diff --git a/apiserver/dora/service/code/pr_filter.py b/apiserver/dora/service/code/pr_filter.py deleted file mode 100644 index 120d16028..000000000 --- a/apiserver/dora/service/code/pr_filter.py +++ /dev/null @@ -1,113 +0,0 @@ -from typing import List, Dict, Any - -from dora.service.settings.configuration_settings import get_settings_service -from dora.service.settings.models import ExcludedPRsSetting -from dora.store.models.code import PRFilter -from dora.store.models.settings.configuration_settings import SettingType -from dora.store.models.settings.enums import EntityType -from dora.utils.regex import regex_list - - -def apply_pr_filter( - pr_filter: Dict = None, - entity_type: EntityType = None, - entity_id: str = None, - setting_types: List[SettingType] = None, -) -> PRFilter: - processed_pr_filter: PRFilter = ParsePRFilterProcessor(pr_filter).apply() - setting_service = get_settings_service() - setting_type_to_settings_map: Dict[SettingType, Any] = {} - - if entity_type and entity_id and setting_types: - setting_type_to_settings_map = setting_service.get_settings_map( - entity_id, setting_types, entity_type - ) - - if entity_type and entity_id and setting_types: - processed_pr_filter = ConfigurationsPRFilterProcessor( - entity_type, - entity_id, - processed_pr_filter, - setting_types, - setting_type_to_settings_map, - ).apply() - return processed_pr_filter - - -class ParsePRFilterProcessor: - def __init__(self, pr_filter: Dict = None): - self.pr_filter = pr_filter or {} - - def apply(self) -> PRFilter: - authors: List[str] = self.__parse_pr_authors() - base_branches: List[str] = self.__parse_pr_base_branches() - repo_filters: Dict[str, Dict] = self.__parse_repo_filters() - - return PRFilter( - authors=authors, - base_branches=base_branches, - repo_filters=repo_filters, - ) - - def __parse_pr_authors(self) -> List[str]: - return self.pr_filter.get("authors") - - def __parse_pr_base_branches(self) -> List[str]: - base_branches: List[str] = self.pr_filter.get("base_branches") - if base_branches: - base_branches: List[str] = regex_list(base_branches) - return base_branches - - def __parse_repo_filters(self) -> Dict[str, Dict]: - repo_filters: Dict[str, Dict] = self.pr_filter.get("repo_filters") - if repo_filters: - for repo_id, repo_filter in repo_filters.items(): - repo_base_branches: List[str] = self.__parse_repo_base_branches( - repo_filter - ) - repo_filters[repo_id]["base_branches"] = repo_base_branches - return repo_filters - - def __parse_repo_base_branches(self, repo_filter: Dict[str, any]) -> List[str]: - repo_base_branches: List[str] = repo_filter.get("base_branches") - if not repo_base_branches: - return [] - repo_base_branches: List[str] = regex_list(repo_base_branches) - return repo_base_branches - - -class ConfigurationsPRFilterProcessor: - def __init__( - self, - entity_type: EntityType, - entity_id: str, - pr_filter: PRFilter, - setting_types: List[SettingType], - setting_type_to_settings_map: Dict[SettingType, Any] = None, - team_member_usernames: List[str] = None, - ): - self.pr_filter = pr_filter or PRFilter() - self.entity_type: EntityType = entity_type - self.entity_id = entity_id - self.setting_types: List[SettingType] = setting_types or [] - self.setting_type_to_settings_map: Dict[SettingType, Any] = ( - setting_type_to_settings_map or {} - ) - self._setting_service = get_settings_service() - self.team_member_usernames = team_member_usernames or [] - - def apply(self) -> PRFilter: - for setting_type in self.setting_types: - setting = self.setting_type_to_settings_map.get( - setting_type, self._setting_service.get_default_setting(setting_type) - ) - if setting_type == SettingType.EXCLUDED_PRS_SETTING: - self._apply_excluded_pr_ids_setting(setting=setting) - - return self.pr_filter - - def _apply_excluded_pr_ids_setting(self, setting: ExcludedPRsSetting): - - self.pr_filter.excluded_pr_ids = ( - self.pr_filter.excluded_pr_ids or [] - ) + setting.excluded_pr_ids diff --git a/apiserver/dora/service/code/sync/__init__.py b/apiserver/dora/service/code/sync/__init__.py deleted file mode 100644 index 621b8a041..000000000 --- a/apiserver/dora/service/code/sync/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .etl_handler import sync_code_repos diff --git a/apiserver/dora/service/code/sync/etl_code_analytics.py b/apiserver/dora/service/code/sync/etl_code_analytics.py deleted file mode 100644 index 942d8b574..000000000 --- a/apiserver/dora/service/code/sync/etl_code_analytics.py +++ /dev/null @@ -1,173 +0,0 @@ -from datetime import timedelta -from typing import List - -from dora.service.code.sync.models import PRPerformance -from dora.store.models.code import ( - PullRequest, - PullRequestEvent, - PullRequestCommit, - PullRequestEventState, - PullRequestState, -) -from dora.utils.time import Interval - - -class CodeETLAnalyticsService: - def create_pr_metrics( - self, - pr: PullRequest, - pr_events: List[PullRequestEvent], - pr_commits: List[PullRequestCommit], - ) -> PullRequest: - if pr.state == PullRequestState.OPEN: - return pr - - pr_performance = self.get_pr_performance(pr, pr_events) - - pr.first_response_time = ( - pr_performance.first_review_time - if pr_performance.first_review_time != -1 - else None - ) - pr.rework_time = ( - pr_performance.rework_time if pr_performance.rework_time != -1 else None - ) - pr.merge_time = ( - pr_performance.merge_time if pr_performance.merge_time != -1 else None - ) - pr.cycle_time = ( - pr_performance.cycle_time if pr_performance.cycle_time != -1 else None - ) - pr.reviewers = list( - {e.actor_username for e in pr_events if e.actor_username != pr.author} - ) - - if pr_commits: - pr.rework_cycles = self.get_rework_cycles(pr, pr_events, pr_commits) - pr_commits.sort(key=lambda x: x.created_at) - first_commit_to_open = pr.created_at - pr_commits[0].created_at - if isinstance(first_commit_to_open, timedelta): - pr.first_commit_to_open = first_commit_to_open.total_seconds() - - return pr - - @staticmethod - def get_pr_performance(pr: PullRequest, pr_events: [PullRequestEvent]): - pr_events.sort(key=lambda x: x.created_at) - first_review = pr_events[0] if pr_events else None - approved_reviews = list( - filter( - lambda x: x.data["state"] == PullRequestEventState.APPROVED.value, - pr_events, - ) - ) - blocking_reviews = list( - filter( - lambda x: x.data["state"] != PullRequestEventState.APPROVED.value, - pr_events, - ) - ) - - if not approved_reviews: - rework_time = -1 - else: - if first_review.data["state"] == PullRequestEventState.APPROVED.value: - rework_time = 0 - else: - rework_time = ( - approved_reviews[0].created_at - first_review.created_at - ).total_seconds() - - if pr.state != PullRequestState.MERGED or not approved_reviews: - merge_time = -1 - else: - merge_time = ( - pr.state_changed_at - approved_reviews[0].created_at - ).total_seconds() - # Prevent garbage state when PR is approved post merging - merge_time = -1 if merge_time < 0 else merge_time - - cycle_time = pr.state_changed_at - pr.created_at - if isinstance(cycle_time, timedelta): - cycle_time = cycle_time.total_seconds() - - return PRPerformance( - first_review_time=(first_review.created_at - pr.created_at).total_seconds() - if first_review - else -1, - rework_time=rework_time, - merge_time=merge_time, - cycle_time=cycle_time if pr.state == PullRequestState.MERGED else -1, - blocking_reviews=len(blocking_reviews), - approving_reviews=len(pr_events) - len(blocking_reviews), - requested_reviews=len(pr.requested_reviews), - ) - - @staticmethod - def get_rework_cycles( - pr: PullRequest, - pr_events: [PullRequestEvent], - pr_commits: [PullRequestCommit], - ) -> int: - - if not pr_events: - return 0 - - if not pr_commits: - return 0 - - pr_events.sort(key=lambda x: x.created_at) - pr_commits.sort(key=lambda x: x.created_at) - - first_blocking_review = None - last_relevant_approval_review = None - pr_reviewers = dict.fromkeys(pr.reviewers, True) - - for pr_event in pr_events: - if ( - pr_event.state != PullRequestEventState.APPROVED.value - and pr_reviewers.get(pr_event.actor_username) - and not first_blocking_review - ): - first_blocking_review = pr_event - - if pr_event.state == PullRequestEventState.APPROVED.value: - last_relevant_approval_review = pr_event - break - - if not first_blocking_review: - return 0 - - if not last_relevant_approval_review: - return 0 - - interval = Interval( - first_blocking_review.created_at - timedelta(seconds=1), - last_relevant_approval_review.created_at, - ) - - pr_commits = list( - filter( - lambda x: x.created_at in interval, - pr_commits, - ) - ) - pr_reviewers = dict.fromkeys(pr.reviewers, True) - blocking_reviews = list( - filter( - lambda x: x.state != PullRequestEventState.APPROVED.value - and x.actor_username != pr.author - and pr_reviewers.get(x.actor_username) - and x.created_at in interval, - pr_events, - ) - ) - all_events = sorted(pr_commits + blocking_reviews, key=lambda x: x.created_at) - rework_cycles = 0 - for curr, next_event in zip(all_events[:-1], all_events[1:]): - if isinstance(curr, type(next_event)): - continue - if isinstance(next_event, PullRequestCommit): - rework_cycles += 1 - - return rework_cycles diff --git a/apiserver/dora/service/code/sync/etl_code_factory.py b/apiserver/dora/service/code/sync/etl_code_factory.py deleted file mode 100644 index 570613af0..000000000 --- a/apiserver/dora/service/code/sync/etl_code_factory.py +++ /dev/null @@ -1,13 +0,0 @@ -from dora.service.code.sync.etl_github_handler import get_github_etl_handler -from dora.service.code.sync.etl_provider_handler import CodeProviderETLHandler -from dora.store.models.code import CodeProvider - - -class CodeETLFactory: - def __init__(self, org_id: str): - self.org_id = org_id - - def __call__(self, provider: str) -> CodeProviderETLHandler: - if provider == CodeProvider.GITHUB.value: - return get_github_etl_handler(self.org_id) - raise NotImplementedError(f"Unknown provider - {provider}") diff --git a/apiserver/dora/service/code/sync/etl_github_handler.py b/apiserver/dora/service/code/sync/etl_github_handler.py deleted file mode 100644 index 1411dabc5..000000000 --- a/apiserver/dora/service/code/sync/etl_github_handler.py +++ /dev/null @@ -1,373 +0,0 @@ -import uuid -from datetime import datetime -from typing import List, Dict, Optional, Tuple, Set - -import pytz -from github.PaginatedList import PaginatedList as GithubPaginatedList -from github.PullRequest import PullRequest as GithubPullRequest -from github.PullRequestReview import PullRequestReview as GithubPullRequestReview -from github.Repository import Repository as GithubRepository - -from dora.exapi.github import GithubApiService -from dora.service.code.sync.etl_code_analytics import CodeETLAnalyticsService -from dora.service.code.sync.etl_provider_handler import CodeProviderETLHandler -from dora.service.code.sync.revert_prs_github_sync import ( - RevertPRsGitHubSyncHandler, - get_revert_prs_github_sync_handler, -) -from dora.store.models import UserIdentityProvider -from dora.store.models.code import ( - OrgRepo, - Bookmark, - PullRequestState, - PullRequest, - PullRequestCommit, - PullRequestEvent, - PullRequestEventType, - PullRequestRevertPRMapping, - CodeProvider, -) -from dora.store.repos.code import CodeRepoService -from dora.store.repos.core import CoreRepoService -from dora.utils.time import time_now, ISO_8601_DATE_FORMAT - -PR_PROCESSING_CHUNK_SIZE = 100 - - -class GithubETLHandler(CodeProviderETLHandler): - def __init__( - self, - org_id: str, - github_api_service: GithubApiService, - code_repo_service: CodeRepoService, - code_etl_analytics_service: CodeETLAnalyticsService, - github_revert_pr_sync_handler: RevertPRsGitHubSyncHandler, - ): - self.org_id: str = org_id - self._api: GithubApiService = github_api_service - self.code_repo_service: CodeRepoService = code_repo_service - self.code_etl_analytics_service: CodeETLAnalyticsService = ( - code_etl_analytics_service - ) - self.github_revert_pr_sync_handler: RevertPRsGitHubSyncHandler = ( - github_revert_pr_sync_handler - ) - self.provider: str = CodeProvider.GITHUB.value - - def check_pat_validity(self) -> bool: - """ - This method checks if the PAT is valid. - :returns: PAT details - :raises: Exception if PAT is invalid - """ - is_valid = self._api.check_pat() - if not is_valid: - raise Exception("Github Personal Access Token is invalid") - return is_valid - - def get_org_repos(self, org_repos: List[OrgRepo]) -> List[OrgRepo]: - """ - This method returns GitHub repos for Org. - :param org_repos: List of OrgRepo objects - :returns: List of GitHub repos as OrgRepo objects - """ - github_repos: List[GithubRepository] = [ - self._api.get_repo(org_repo.org_name, org_repo.name) - for org_repo in org_repos - ] - return [ - self._process_github_repo(org_repos, github_repo) - for github_repo in github_repos - ] - - def get_repo_pull_requests_data( - self, org_repo: OrgRepo, bookmark: Bookmark - ) -> Tuple[List[PullRequest], List[PullRequestCommit], List[PullRequestEvent]]: - """ - This method returns all pull requests, their Commits and Events of a repo. - :param org_repo: OrgRepo object to get pull requests for - :param bookmark: Bookmark date to get all pull requests after this date - :return: Pull requests, their commits and events - """ - github_repo: GithubRepository = self._api.get_repo( - org_repo.org_name, org_repo.name - ) - github_pull_requests: GithubPaginatedList = self._api.get_pull_requests( - github_repo - ) - - prs_to_process = [] - bookmark_time = datetime.fromisoformat(bookmark.bookmark) - for page in range( - 0, github_pull_requests.totalCount // PR_PROCESSING_CHUNK_SIZE + 1, 1 - ): - prs = github_pull_requests.get_page(page) - if not prs: - break - - if prs[-1].updated_at.astimezone(tz=pytz.UTC) <= bookmark_time: - prs_to_process += [ - pr - for pr in prs - if pr.updated_at.astimezone(tz=pytz.UTC) > bookmark_time - ] - break - - prs_to_process += prs - - filtered_prs: List = [] - for pr in prs_to_process: - state_changed_at = pr.merged_at if pr.merged_at else pr.closed_at - if ( - pr.state.upper() != PullRequestState.OPEN.value - and state_changed_at.astimezone(tz=pytz.UTC) < bookmark_time - ): - continue - if pr not in filtered_prs: - filtered_prs.append(pr) - - filtered_prs = filtered_prs[::-1] - - if not filtered_prs: - print("Nothing to process 🎉") - return [], [], [] - - pull_requests: List[PullRequest] = [] - pr_commits: List[PullRequestCommit] = [] - pr_events: List[PullRequestEvent] = [] - prs_added: Set[int] = set() - - for github_pr in filtered_prs: - if github_pr.number in prs_added: - continue - - pr_model, event_models, pr_commit_models = self.process_pr( - str(org_repo.id), github_pr - ) - pull_requests.append(pr_model) - pr_events += event_models - pr_commits += pr_commit_models - prs_added.add(github_pr.number) - - return pull_requests, pr_commits, pr_events - - def process_pr( - self, repo_id: str, pr: GithubPullRequest - ) -> Tuple[PullRequest, List[PullRequestEvent], List[PullRequestCommit]]: - pr_model: Optional[PullRequest] = self.code_repo_service.get_repo_pr_by_number( - repo_id, pr.number - ) - pr_event_model_list: List[ - PullRequestEvent - ] = self.code_repo_service.get_pr_events(pr_model) - pr_commits_model_list: List = [] - - reviews: List[GithubPullRequestReview] = list(self._api.get_pr_reviews(pr)) - pr_model: PullRequest = self._to_pr_model(pr, pr_model, repo_id, len(reviews)) - pr_events_model_list: List[PullRequestEvent] = self._to_pr_events( - reviews, pr_model, pr_event_model_list - ) - if pr.merged_at: - commits: List[Dict] = list( - map( - lambda x: x.__dict__["_rawData"], list(self._api.get_pr_commits(pr)) - ) - ) - pr_commits_model_list: List[PullRequestCommit] = self._to_pr_commits( - commits, pr_model - ) - - pr_model = self.code_etl_analytics_service.create_pr_metrics( - pr_model, pr_events_model_list, pr_commits_model_list - ) - - return pr_model, pr_events_model_list, pr_commits_model_list - - def get_revert_prs_mapping( - self, prs: List[PullRequest] - ) -> List[PullRequestRevertPRMapping]: - return self.github_revert_pr_sync_handler(prs) - - def _process_github_repo( - self, org_repos: List[OrgRepo], github_repo: GithubRepository - ) -> OrgRepo: - - repo_idempotency_key_id_map = { - org_repo.idempotency_key: str(org_repo.id) for org_repo in org_repos - } - - org_repo = OrgRepo( - id=repo_idempotency_key_id_map.get(str(github_repo.id), uuid.uuid4()), - org_id=self.org_id, - name=github_repo.name, - provider=self.provider, - org_name=github_repo.organization.login, - default_branch=github_repo.default_branch, - language=github_repo.language, - contributors=self._api.get_repo_contributors(github_repo), - idempotency_key=str(github_repo.id), - slug=github_repo.name, - updated_at=time_now(), - ) - return org_repo - - def _to_pr_model( - self, - pr: GithubPullRequest, - pr_model: Optional[PullRequest], - repo_id: str, - review_comments: int = 0, - ) -> PullRequest: - state = self._get_state(pr) - pr_id = pr_model.id if pr_model else uuid.uuid4() - state_changed_at = None - if state != PullRequestState.OPEN: - state_changed_at = ( - pr.merged_at.astimezone(pytz.UTC) - if pr.merged_at - else pr.closed_at.astimezone(pytz.UTC) - ) - - merge_commit_sha: Optional[str] = self._get_merge_commit_sha(pr.raw_data, state) - - return PullRequest( - id=pr_id, - number=str(pr.number), - title=pr.title, - url=pr.html_url, - created_at=pr.created_at.astimezone(pytz.UTC), - updated_at=pr.updated_at.astimezone(pytz.UTC), - state_changed_at=state_changed_at, - state=state, - base_branch=pr.base.ref, - head_branch=pr.head.ref, - author=pr.user.login, - repo_id=repo_id, - data=pr.raw_data, - requested_reviews=[r["login"] for r in pr.raw_data["requested_reviewers"]], - meta=dict( - code_stats=dict( - commits=pr.commits, - additions=pr.additions, - deletions=pr.deletions, - changed_files=pr.changed_files, - comments=review_comments, - ), - user_profile=dict(username=pr.user.login), - ), - provider=UserIdentityProvider.GITHUB.value, - merge_commit_sha=merge_commit_sha, - ) - - @staticmethod - def _get_merge_commit_sha(raw_data: Dict, state: PullRequestState) -> Optional[str]: - if state != PullRequestState.MERGED: - return None - - merge_commit_sha = raw_data.get("merge_commit_sha") - - return merge_commit_sha - - @staticmethod - def _get_state(pr: GithubPullRequest) -> PullRequestState: - if pr.merged_at: - return PullRequestState.MERGED - if pr.closed_at: - return PullRequestState.CLOSED - - return PullRequestState.OPEN - - @staticmethod - def _to_pr_events( - reviews: [GithubPullRequestReview], - pr_model: PullRequest, - pr_events_model: [PullRequestEvent], - ) -> List[PullRequestEvent]: - pr_events: List[PullRequestEvent] = [] - pr_event_id_map = {event.idempotency_key: event.id for event in pr_events_model} - - for review in reviews: - if not review.submitted_at: - continue # Discard incomplete reviews - - actor = review.raw_data.get("user", {}) - username = actor.get("login", "") if actor else "" - - pr_events.append( - PullRequestEvent( - id=pr_event_id_map.get(str(review.id), uuid.uuid4()), - pull_request_id=str(pr_model.id), - type=PullRequestEventType.REVIEW.value, - data=review.raw_data, - created_at=review.submitted_at.astimezone(pytz.UTC), - idempotency_key=str(review.id), - org_repo_id=pr_model.repo_id, - actor_username=username, - ) - ) - return pr_events - - def _to_pr_commits( - self, - commits: List[Dict], - pr_model: PullRequest, - ) -> List[PullRequestCommit]: - """ - Sample commit - - { - 'sha': '123456789098765', - 'commit': { - 'committer': {'name': 'abc', 'email': 'abc@midd.com', 'date': '2022-06-29T10:53:15Z'}, - 'message': '[abc 315] avoid mapping edit state', - } - 'author': {'login': 'abc', 'id': 95607047, 'node_id': 'abc', 'avatar_url': ''}, - 'html_url': 'https://github.com/123456789098765', - } - """ - pr_commits: List[PullRequestCommit] = [] - - for commit in commits: - pr_commits.append( - PullRequestCommit( - hash=commit["sha"], - pull_request_id=str(pr_model.id), - url=commit["html_url"], - data=commit, - message=commit["commit"]["message"], - author=commit["author"]["login"] - if commit.get("author") - else commit["commit"].get("committer", {}).get("email", ""), - created_at=self._dt_from_github_dt_string( - commit["commit"]["committer"]["date"] - ), - org_repo_id=pr_model.repo_id, - ) - ) - return pr_commits - - @staticmethod - def _dt_from_github_dt_string(dt_string: str) -> datetime: - dt_without_timezone = datetime.strptime(dt_string, ISO_8601_DATE_FORMAT) - return dt_without_timezone.replace(tzinfo=pytz.UTC) - - -def get_github_etl_handler(org_id: str) -> GithubETLHandler: - def _get_access_token(): - core_repo_service = CoreRepoService() - access_token = core_repo_service.get_access_token( - org_id, UserIdentityProvider.GITHUB - ) - if not access_token: - raise Exception( - f"Access token not found for org {org_id} and provider {UserIdentityProvider.GITHUB.value}" - ) - return access_token - - return GithubETLHandler( - org_id, - GithubApiService(_get_access_token()), - CodeRepoService(), - CodeETLAnalyticsService(), - get_revert_prs_github_sync_handler(), - ) diff --git a/apiserver/dora/service/code/sync/etl_handler.py b/apiserver/dora/service/code/sync/etl_handler.py deleted file mode 100644 index 52ae79497..000000000 --- a/apiserver/dora/service/code/sync/etl_handler.py +++ /dev/null @@ -1,125 +0,0 @@ -from datetime import datetime, timedelta -from typing import List - -import pytz - -from dora.service.code.integration import get_code_integration_service -from dora.service.code.sync.etl_code_factory import ( - CodeProviderETLHandler, - CodeETLFactory, -) -from dora.service.merge_to_deploy_broker import ( - get_merge_to_deploy_broker_utils_service, - MergeToDeployBrokerUtils, -) -from dora.store.models.code import OrgRepo, BookmarkType, Bookmark, PullRequest -from dora.store.repos.code import CodeRepoService -from dora.utils.log import LOG - - -class CodeETLHandler: - def __init__( - self, - code_repo_service: CodeRepoService, - etl_service: CodeProviderETLHandler, - mtd_broker: MergeToDeployBrokerUtils, - ): - self.code_repo_service = code_repo_service - self.etl_service = etl_service - self.mtd_broker = mtd_broker - - def sync_org_repos(self, org_id: str): - if not self.etl_service.check_pat_validity(): - LOG.error("Invalid PAT for code provider") - return - org_repos: List[OrgRepo] = self._sync_org_repos(org_id) - for org_repo in org_repos: - try: - self._sync_repo_pull_requests_data(org_repo) - except Exception as e: - LOG.error( - f"Error syncing pull requests for repo {org_repo.name}: {str(e)}" - ) - continue - - def _sync_org_repos(self, org_id: str) -> List[OrgRepo]: - try: - org_repos = self.code_repo_service.get_active_org_repos(org_id) - self.etl_service.get_org_repos(org_repos) - self.code_repo_service.update_org_repos(org_repos) - return org_repos - except Exception as e: - LOG.error(f"Error syncing org repos for org {org_id}: {str(e)}") - raise e - - def _sync_repo_pull_requests_data(self, org_repo: OrgRepo) -> None: - try: - bookmark: Bookmark = self.__get_org_repo_bookmark(org_repo) - ( - pull_requests, - pull_request_commits, - pull_request_events, - ) = self.etl_service.get_repo_pull_requests_data(org_repo, bookmark) - self.code_repo_service.save_pull_requests_data( - pull_requests, pull_request_commits, pull_request_events - ) - if not pull_requests: - return - bookmark.bookmark = ( - pull_requests[-1].state_changed_at.astimezone(tz=pytz.UTC).isoformat() - ) - self.code_repo_service.update_org_repo_bookmark(bookmark) - self.mtd_broker.pushback_merge_to_deploy_bookmark( - str(org_repo.id), pull_requests - ) - self.__sync_revert_prs_mapping(org_repo, pull_requests) - except Exception as e: - LOG.error(f"Error syncing pull requests for repo {org_repo.name}: {str(e)}") - raise e - - def __sync_revert_prs_mapping( - self, org_repo: OrgRepo, prs: List[PullRequest] - ) -> None: - try: - revert_prs_mapping = self.etl_service.get_revert_prs_mapping(prs) - self.code_repo_service.save_revert_pr_mappings(revert_prs_mapping) - except Exception as e: - LOG.error(f"Error syncing revert PRs for repo {org_repo.name}: {str(e)}") - raise e - - def __get_org_repo_bookmark(self, org_repo: OrgRepo, default_sync_days: int = 31): - bookmark = self.code_repo_service.get_org_repo_bookmark( - org_repo, BookmarkType.PR - ) - if not bookmark: - default_pr_bookmark = datetime.now().astimezone(tz=pytz.UTC) - timedelta( - days=default_sync_days - ) - bookmark = Bookmark( - repo_id=org_repo.id, - type=BookmarkType.PR.value, - bookmark=default_pr_bookmark.isoformat(), - ) - return bookmark - - -def sync_code_repos(org_id: str): - code_providers: List[str] = get_code_integration_service().get_org_providers(org_id) - if not code_providers: - LOG.info(f"No code integrations found for org {org_id}") - return - etl_factory = CodeETLFactory(org_id) - - for provider in code_providers: - try: - code_etl_handler = CodeETLHandler( - CodeRepoService(), - etl_factory(provider), - get_merge_to_deploy_broker_utils_service(), - ) - code_etl_handler.sync_org_repos(org_id) - LOG.info(f"Synced org repos for provider {provider}") - except Exception as e: - LOG.error(f"Error syncing org repos for provider {provider}: {str(e)}") - continue - LOG.info(f"Synced all org repos for org {org_id}") diff --git a/apiserver/dora/service/code/sync/etl_provider_handler.py b/apiserver/dora/service/code/sync/etl_provider_handler.py deleted file mode 100644 index c378b958c..000000000 --- a/apiserver/dora/service/code/sync/etl_provider_handler.py +++ /dev/null @@ -1,53 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List, Tuple - -from dora.store.models.code import ( - OrgRepo, - PullRequest, - PullRequestCommit, - PullRequestEvent, - PullRequestRevertPRMapping, - Bookmark, -) - - -class CodeProviderETLHandler(ABC): - @abstractmethod - def check_pat_validity(self) -> bool: - """ - This method checks if the PAT is valid. - :return: PAT details - :raises: Exception if PAT is invalid - """ - pass - - @abstractmethod - def get_org_repos(self, org_repos: List[OrgRepo]) -> List[OrgRepo]: - """ - This method returns all repos from provider that are in sync and available for the provider in given access token. - :return: List of repos as OrgRepo objects - """ - pass - - @abstractmethod - def get_repo_pull_requests_data( - self, org_repo: OrgRepo, bookmark: Bookmark - ) -> Tuple[List[PullRequest], List[PullRequestCommit], List[PullRequestEvent]]: - """ - This method returns all pull requests, their Commits and Events of a repo. After the bookmark date. - :param org_repo: OrgRepo object to get pull requests for - :param bookmark: Bookmark object to get all pull requests after this date - :return: Pull requests sorted by state_changed_at date, their commits and events - """ - pass - - @abstractmethod - def get_revert_prs_mapping( - self, prs: List[PullRequest] - ) -> List[PullRequestRevertPRMapping]: - """ - This method processes all PRs and returns the mapping of revert PRs with source PRs. - :param prs: List of PRs to process - :return: List of PullRequestRevertPRMapping objects - """ - pass diff --git a/apiserver/dora/service/code/sync/models.py b/apiserver/dora/service/code/sync/models.py deleted file mode 100644 index 69b2176dc..000000000 --- a/apiserver/dora/service/code/sync/models.py +++ /dev/null @@ -1,19 +0,0 @@ -from dataclasses import dataclass - - -@dataclass -class PRPerformance: - first_commit_to_open: int = -1 - first_review_time: int = -1 - rework_time: int = -1 - merge_time: int = -1 - merge_to_deploy: int = -1 - cycle_time: int = -1 - blocking_reviews: int = -1 - approving_reviews: int = -1 - requested_reviews: int = -1 - prs_authored_count: int = -1 - additions: int = -1 - deletions: int = -1 - rework_cycles: int = -1 - lead_time: int = -1 diff --git a/apiserver/dora/service/code/sync/revert_prs_github_sync.py b/apiserver/dora/service/code/sync/revert_prs_github_sync.py deleted file mode 100644 index 74ddefae9..000000000 --- a/apiserver/dora/service/code/sync/revert_prs_github_sync.py +++ /dev/null @@ -1,185 +0,0 @@ -import re -from datetime import datetime -from typing import List, Set, Dict, Optional - -from dora.store.models.code import ( - PullRequest, - PullRequestRevertPRMapping, - PullRequestRevertPRMappingActorType, -) -from dora.store.repos.code import CodeRepoService -from dora.utils.time import time_now - - -class RevertPRsGitHubSyncHandler: - def __init__( - self, - code_repo_service: CodeRepoService, - ): - self.code_repo_service = code_repo_service - - def __call__(self, *args, **kwargs): - return self.process_revert_prs(*args, **kwargs) - - def process_revert_prs( - self, prs: List[PullRequest] - ) -> List[PullRequestRevertPRMapping]: - revert_prs: List[PullRequest] = [] - original_prs: List[PullRequest] = [] - - for pr in prs: - pr_number = ( - self._get_revert_pr_number(pr.head_branch) if pr.head_branch else None - ) - if pr_number is None: - original_prs.append(pr) - else: - revert_prs.append(pr) - - mappings_of_revert_prs = self._get_revert_pr_mapping_for_revert_prs(revert_prs) - mappings_of_original_prs = self._get_revert_pr_mapping_for_original_prs( - original_prs - ) - revert_pr_mappings = set(mappings_of_original_prs + mappings_of_revert_prs) - - return list(revert_pr_mappings) - - def _get_revert_pr_mapping_for_original_prs( - self, prs: List[PullRequest] - ) -> List[PullRequestRevertPRMapping]: - """ - This function takes a list of PRs and for each PR it tries to - find if that pr has been reverted and by which PR. It is done - by taking repo_id and the pr_number and searching for the - string 'revert-[pr-number]' in the head branch. - """ - - repo_ids: Set[str] = set() - repo_id_to_pr_number_to_id_map: Dict[str, Dict[str, str]] = {} - pr_numbers_match_strings: List[str] = [] - - for pr in prs: - pr_numbers_match_strings.append(f"revert-{pr.number}") - repo_ids.add(str(pr.repo_id)) - - if str(pr.repo_id) not in repo_id_to_pr_number_to_id_map: - repo_id_to_pr_number_to_id_map[str(pr.repo_id)] = {} - - repo_id_to_pr_number_to_id_map[str(pr.repo_id)][str(pr.number)] = pr.id - - if len(pr_numbers_match_strings) == 0: - return [] - - revert_prs: List[ - PullRequest - ] = self.code_repo_service.get_prs_by_head_branch_match_strings( - list(repo_ids), pr_numbers_match_strings - ) - - revert_pr_mappings: List[PullRequestRevertPRMapping] = [] - - for rev_pr in revert_prs: - original_pr_number = self._get_revert_pr_number(rev_pr.head_branch) - if original_pr_number is None: - continue - - repo_key_exists = repo_id_to_pr_number_to_id_map.get(str(rev_pr.repo_id)) - if repo_key_exists is None: - continue - - original_pr_id = repo_id_to_pr_number_to_id_map[str(rev_pr.repo_id)].get( - original_pr_number - ) - if original_pr_id is None: - continue - - revert_pr_mp = PullRequestRevertPRMapping( - pr_id=rev_pr.id, - actor_type=PullRequestRevertPRMappingActorType.SYSTEM, - actor=None, - reverted_pr=original_pr_id, - updated_at=time_now(), - ) - revert_pr_mappings.append(revert_pr_mp) - - return revert_pr_mappings - - def _get_revert_pr_mapping_for_revert_prs( - self, prs: List[PullRequest] - ) -> List[PullRequestRevertPRMapping]: - """ - This function takes a list of pull requests and for each pull request - checks if it is a revert pr or not. If it is a revert pr it tries to - create a mapping of that revert pr with the reverted pr and then returns - a list of those mappings - """ - - revert_pr_numbers: List[str] = [] - repo_ids: Set[str] = set() - repo_id_to_pr_number_to_id_map: Dict[str, Dict[str, str]] = {} - - for pr in prs: - revert_pr_number = self._get_revert_pr_number(pr.head_branch) - if revert_pr_number is None: - continue - - revert_pr_numbers.append(revert_pr_number) - repo_ids.add(str(pr.repo_id)) - - if str(pr.repo_id) not in repo_id_to_pr_number_to_id_map: - repo_id_to_pr_number_to_id_map[str(pr.repo_id)] = {} - - repo_id_to_pr_number_to_id_map[str(pr.repo_id)][ - str(revert_pr_number) - ] = pr.id - - if len(revert_pr_numbers) == 0: - return [] - - reverted_prs: List[ - PullRequest - ] = self.code_repo_service.get_reverted_prs_by_numbers( - list(repo_ids), revert_pr_numbers - ) - - revert_pr_mappings: List[PullRequestRevertPRMapping] = [] - for rev_pr in reverted_prs: - repo_key_exists = repo_id_to_pr_number_to_id_map.get(str(rev_pr.repo_id)) - if repo_key_exists is None: - continue - - original_pr_id = repo_id_to_pr_number_to_id_map[str(rev_pr.repo_id)].get( - str(rev_pr.number) - ) - if original_pr_id is None: - continue - - revert_pr_mp = PullRequestRevertPRMapping( - pr_id=original_pr_id, - actor_type=PullRequestRevertPRMappingActorType.SYSTEM, - actor=None, - reverted_pr=rev_pr.id, - updated_at=datetime.now(), - ) - revert_pr_mappings.append(revert_pr_mp) - - return revert_pr_mappings - - def _get_revert_pr_number(self, head_branch: str) -> Optional[str]: - """ - Function to match the regex pattern "revert-[pr-num]-[branch-name]" and - return the PR number for GitHub. - """ - pattern = r"revert-(\d+)-\w+" - - match = re.search(pattern, head_branch) - - if match: - pr_num = match.group(1) - return pr_num - else: - return None - - -def get_revert_prs_github_sync_handler() -> RevertPRsGitHubSyncHandler: - return RevertPRsGitHubSyncHandler(CodeRepoService()) diff --git a/apiserver/dora/service/core/teams.py b/apiserver/dora/service/core/teams.py deleted file mode 100644 index c9b233298..000000000 --- a/apiserver/dora/service/core/teams.py +++ /dev/null @@ -1,35 +0,0 @@ -from typing import List, Optional -from dora.store.models.core.teams import Team -from dora.store.repos.core import CoreRepoService - - -class TeamService: - def __init__(self, core_repo_service: CoreRepoService): - self._core_repo_service = core_repo_service - - def get_team(self, team_id: str) -> Optional[Team]: - return self._core_repo_service.get_team(team_id) - - def delete_team(self, team_id: str) -> Optional[Team]: - return self._core_repo_service.delete_team(team_id) - - def create_team(self, org_id: str, name: str, member_ids: List[str] = None) -> Team: - return self._core_repo_service.create_team(org_id, name, member_ids or []) - - def update_team( - self, team_id: str, name: str = None, member_ids: List[str] = None - ) -> Team: - - team = self._core_repo_service.get_team(team_id) - - if name is not None: - team.name = name - - if member_ids is not None: - team.member_ids = member_ids - - return self._core_repo_service.update_team(team) - - -def get_team_service(): - return TeamService(CoreRepoService()) diff --git a/apiserver/dora/service/deployments/__init__.py b/apiserver/dora/service/deployments/__init__.py deleted file mode 100644 index 7ef06b7be..000000000 --- a/apiserver/dora/service/deployments/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .deployment_pr_mapper import DeploymentPRMapperService diff --git a/apiserver/dora/service/deployments/analytics.py b/apiserver/dora/service/deployments/analytics.py deleted file mode 100644 index f4ef19f00..000000000 --- a/apiserver/dora/service/deployments/analytics.py +++ /dev/null @@ -1,257 +0,0 @@ -from collections import defaultdict -from datetime import datetime -from typing import List, Dict, Tuple - -from dora.utils.dict import ( - get_average_of_dict_values, - get_key_to_count_map_from_key_to_list_map, -) - -from .deployment_service import DeploymentsService, get_deployments_service -from dora.store.models.code.filter import PRFilter -from dora.store.models.code.pull_requests import PullRequest -from dora.store.models.code.repository import TeamRepos -from dora.store.models.code.workflows.filter import WorkflowFilter -from dora.service.deployments.models.models import ( - Deployment, - DeploymentFrequencyMetrics, -) - -from dora.store.repos.code import CodeRepoService -from dora.utils.time import Interval, generate_expanded_buckets - - -class DeploymentAnalyticsService: - def __init__( - self, - deployments_service: DeploymentsService, - code_repo_service: CodeRepoService, - ): - self.deployments_service = deployments_service - self.code_repo_service = code_repo_service - - def get_team_successful_deployments_in_interval_with_related_prs( - self, - team_id: str, - interval: Interval, - pr_filter: PRFilter, - workflow_filter: WorkflowFilter, - ) -> Dict[str, List[Dict[Deployment, List[PullRequest]]]]: - """ - Retrieves successful deployments within the specified interval for a given team, - along with related pull requests. Returns A dictionary mapping repository IDs to lists of deployments along with related pull requests. Each deployment is associated with a list of pull requests that contributed to it. - """ - - deployments: List[ - Deployment - ] = self.deployments_service.get_team_successful_deployments_in_interval( - team_id, interval, pr_filter, workflow_filter - ) - - team_repos: List[TeamRepos] = self._get_team_repos_by_team_id(team_id) - repo_ids: List[str] = [str(team_repo.org_repo_id) for team_repo in team_repos] - - pull_requests: List[ - PullRequest - ] = self.code_repo_service.get_prs_merged_in_interval( - repo_ids, interval, pr_filter - ) - - repo_id_branch_to_pr_list_map: Dict[ - Tuple[str, str], List[PullRequest] - ] = self._map_prs_to_repo_id_and_base_branch(pull_requests) - repo_id_branch_to_deployments_map: Dict[ - Tuple[str, str], List[Deployment] - ] = self._map_deployments_to_repo_id_and_head_branch(deployments) - - repo_id_to_deployments_with_pr_map: Dict[ - str, Dict[Deployment, List[PullRequest]] - ] = defaultdict(dict) - - for ( - repo_id, - base_branch, - ), deployments in repo_id_branch_to_deployments_map.items(): - relevant_prs: List[PullRequest] = repo_id_branch_to_pr_list_map.get( - (repo_id, base_branch), [] - ) - deployments_pr_map: Dict[ - Deployment, List[PullRequest] - ] = self._map_prs_to_deployments(relevant_prs, deployments) - - repo_id_to_deployments_with_pr_map[repo_id].update(deployments_pr_map) - - return repo_id_to_deployments_with_pr_map - - def get_team_deployment_frequency_metrics( - self, - team_id: str, - interval: Interval, - pr_filter: PRFilter, - workflow_filter: WorkflowFilter, - ) -> DeploymentFrequencyMetrics: - - team_successful_deployments = ( - self.deployments_service.get_team_successful_deployments_in_interval( - team_id, interval, pr_filter, workflow_filter - ) - ) - - return self._get_deployment_frequency_metrics( - team_successful_deployments, interval - ) - - def get_weekly_deployment_frequency_trends( - self, - team_id: str, - interval: Interval, - pr_filter: PRFilter, - workflow_filter: WorkflowFilter, - ) -> Dict[datetime, int]: - - team_successful_deployments = ( - self.deployments_service.get_team_successful_deployments_in_interval( - team_id, interval, pr_filter, workflow_filter - ) - ) - - team_weekly_deployments = generate_expanded_buckets( - team_successful_deployments, interval, "conducted_at", "weekly" - ) - - return get_key_to_count_map_from_key_to_list_map(team_weekly_deployments) - - def _map_prs_to_repo_id_and_base_branch( - self, pull_requests: List[PullRequest] - ) -> Dict[Tuple[str, str], List[PullRequest]]: - repo_id_branch_pr_map: Dict[Tuple[str, str], List[PullRequest]] = defaultdict( - list - ) - for pr in pull_requests: - repo_id = str(pr.repo_id) - base_branch = pr.base_branch - repo_id_branch_pr_map[(repo_id, base_branch)].append(pr) - return repo_id_branch_pr_map - - def _map_deployments_to_repo_id_and_head_branch( - self, deployments: List[Deployment] - ) -> Dict[Tuple[str, str], List[Deployment]]: - repo_id_branch_deployments_map: Dict[ - Tuple[str, str], List[Deployment] - ] = defaultdict(list) - for deployment in deployments: - repo_id = str(deployment.repo_id) - head_branch = deployment.head_branch - repo_id_branch_deployments_map[(repo_id, head_branch)].append(deployment) - return repo_id_branch_deployments_map - - def _map_prs_to_deployments( - self, pull_requests: List[PullRequest], deployments: List[Deployment] - ) -> Dict[Deployment, List[PullRequest]]: - """ - Maps the pull requests to the deployments they were included in. - This method takes a sorted list of pull requests and a sorted list of deployments and returns a dictionary - """ - pr_count = 0 - deployment_count = 0 - deployment_pr_map = defaultdict( - list, {deployment: [] for deployment in deployments} - ) - - while pr_count < len(pull_requests) and deployment_count < len(deployments): - pr = pull_requests[pr_count] - deployment = deployments[deployment_count] - - # Check if the PR was merged before or at the same time as the deployment - if pr.state_changed_at <= deployment.conducted_at: - deployment_pr_map[deployment].append(pr) - pr_count += 1 - else: - deployment_count += 1 - - return deployment_pr_map - - def _get_team_repos_by_team_id(self, team_id: str) -> List[TeamRepos]: - return self.code_repo_service.get_active_team_repos_by_team_id(team_id) - - def _get_deployment_frequency_from_date_to_deployment_map( - self, date_to_deployment_map: Dict[datetime, List[Deployment]] - ) -> int: - """ - This method takes a dict of datetime representing (day/week/month) to Deployments and returns avg deployment frequency - """ - - date_to_deployment_count_map: Dict[ - datetime, int - ] = get_key_to_count_map_from_key_to_list_map(date_to_deployment_map) - - return get_average_of_dict_values(date_to_deployment_count_map) - - def _get_deployment_frequency_metrics( - self, successful_deployments: List[Deployment], interval: Interval - ) -> DeploymentFrequencyMetrics: - - successful_deployments = list( - filter( - lambda x: x.conducted_at >= interval.from_time - and x.conducted_at <= interval.to_time, - successful_deployments, - ) - ) - - team_daily_deployments = generate_expanded_buckets( - successful_deployments, interval, "conducted_at", "daily" - ) - team_weekly_deployments = generate_expanded_buckets( - successful_deployments, interval, "conducted_at", "weekly" - ) - team_monthly_deployments = generate_expanded_buckets( - successful_deployments, interval, "conducted_at", "monthly" - ) - - daily_deployment_frequency = ( - self._get_deployment_frequency_from_date_to_deployment_map( - team_daily_deployments - ) - ) - - weekly_deployment_frequency = ( - self._get_deployment_frequency_from_date_to_deployment_map( - team_weekly_deployments - ) - ) - - monthly_deployment_frequency = ( - self._get_deployment_frequency_from_date_to_deployment_map( - team_monthly_deployments - ) - ) - - return DeploymentFrequencyMetrics( - len(successful_deployments), - daily_deployment_frequency, - weekly_deployment_frequency, - monthly_deployment_frequency, - ) - - def _get_weekly_deployment_frequency_trends( - self, successful_deployments: List[Deployment], interval: Interval - ) -> Dict[datetime, int]: - - successful_deployments = list( - filter( - lambda x: x.conducted_at >= interval.from_time - and x.conducted_at <= interval.to_time, - successful_deployments, - ) - ) - - team_weekly_deployments = generate_expanded_buckets( - successful_deployments, interval, "conducted_at", "weekly" - ) - - return get_key_to_count_map_from_key_to_list_map(team_weekly_deployments) - - -def get_deployment_analytics_service() -> DeploymentAnalyticsService: - return DeploymentAnalyticsService(get_deployments_service(), CodeRepoService()) diff --git a/apiserver/dora/service/deployments/deployment_pr_mapper.py b/apiserver/dora/service/deployments/deployment_pr_mapper.py deleted file mode 100644 index fa4971e87..000000000 --- a/apiserver/dora/service/deployments/deployment_pr_mapper.py +++ /dev/null @@ -1,79 +0,0 @@ -from collections import defaultdict -from datetime import datetime -from queue import Queue -from typing import List -from dora.store.models.code.enums import PullRequestState - -from dora.store.models.code.pull_requests import PullRequest -from dora.service.deployments.models.models import Deployment - - -class DeploymentPRGraph: - def __init__(self): - self._nodes = set() - self._adj_list = defaultdict(list) - self._last_change_deployed_for_branch = {} - - def add_edge(self, base_branch, head_branch, pr: PullRequest): - self._nodes.add(base_branch) - - self._adj_list[base_branch].append((head_branch, pr)) - if head_branch not in self._last_change_deployed_for_branch: - self._last_change_deployed_for_branch[head_branch] = pr.state_changed_at - - self._last_change_deployed_for_branch[head_branch] = max( - self._last_change_deployed_for_branch[head_branch], pr.state_changed_at - ) - - def get_edges(self, base_branch: str): - return self._adj_list[base_branch] - - def get_all_prs_for_root(self, base_branch) -> List[PullRequest]: - if base_branch not in self._nodes: - return [] - - prs = set() - q = Queue() - visited = defaultdict(bool) - q.put(base_branch) - - while not q.empty(): - front = q.get() - if visited[front]: - continue - - visited[front] = True - for edge in self.get_edges(front): - branch, pr = edge - if self._is_pr_merged_post_last_change(pr=pr, base_branch=front): - continue - - q.put(branch) - prs.add(pr) - - return list(prs) - - def _is_pr_merged_post_last_change(self, pr: PullRequest, base_branch: str): - return pr.state_changed_at > self._last_change_deployed_for_branch[base_branch] - - def set_root_deployment_time(self, root_branch, deployment_time: datetime): - self._last_change_deployed_for_branch[root_branch] = deployment_time - - -class DeploymentPRMapperService: - def get_all_prs_deployed( - self, prs: List[PullRequest], deployment: Deployment - ) -> List[PullRequest]: - - branch_graph = DeploymentPRGraph() - branch_graph.set_root_deployment_time( - deployment.head_branch, deployment.conducted_at - ) - - for pr in prs: - if pr.state != PullRequestState.MERGED: - continue - - branch_graph.add_edge(pr.base_branch, pr.head_branch, pr) - - return branch_graph.get_all_prs_for_root(deployment.head_branch) diff --git a/apiserver/dora/service/deployments/deployment_service.py b/apiserver/dora/service/deployments/deployment_service.py deleted file mode 100644 index 4278615c1..000000000 --- a/apiserver/dora/service/deployments/deployment_service.py +++ /dev/null @@ -1,171 +0,0 @@ -from typing import List, Tuple -from dora.store.models.code.workflows import RepoWorkflowType, RepoWorkflow - -from .factory import get_deployments_factory -from .deployments_factory_service import DeploymentsFactoryService -from dora.store.models.code.filter import PRFilter -from dora.store.models.code.repository import TeamRepos -from dora.store.models.code.workflows.filter import WorkflowFilter -from dora.service.deployments.models.models import Deployment, DeploymentType - -from dora.store.repos.code import CodeRepoService -from dora.store.repos.workflows import WorkflowRepoService -from dora.utils.time import Interval - - -class DeploymentsService: - def __init__( - self, - code_repo_service: CodeRepoService, - workflow_repo_service: WorkflowRepoService, - workflow_based_deployments_service: DeploymentsFactoryService, - pr_based_deployments_service: DeploymentsFactoryService, - ): - self.code_repo_service = code_repo_service - self.workflow_repo_service = workflow_repo_service - self.workflow_based_deployments_service = workflow_based_deployments_service - self.pr_based_deployments_service = pr_based_deployments_service - - def get_team_successful_deployments_in_interval( - self, - team_id: str, - interval: Interval, - pr_filter: PRFilter = None, - workflow_filter: WorkflowFilter = None, - ) -> List[Deployment]: - team_repos = self._get_team_repos_by_team_id(team_id) - ( - team_repos_using_workflow_deployments, - team_repos_using_pr_deployments, - ) = self.get_filtered_team_repos_by_deployment_config(team_repos) - - deployments_using_workflow = self.workflow_based_deployments_service.get_repos_successful_deployments_in_interval( - self._get_repo_ids_from_team_repos(team_repos_using_workflow_deployments), - interval, - workflow_filter, - ) - deployments_using_pr = self.pr_based_deployments_service.get_repos_successful_deployments_in_interval( - self._get_repo_ids_from_team_repos(team_repos_using_pr_deployments), - interval, - pr_filter, - ) - - deployments: List[Deployment] = ( - deployments_using_workflow + deployments_using_pr - ) - sorted_deployments = self._sort_deployments_by_date(deployments) - - return sorted_deployments - - def get_filtered_team_repos_with_workflow_configured_deployments( - self, team_repos: List[TeamRepos] - ) -> List[TeamRepos]: - """ - Get team repos with workflow deployments configured. - That is the repo has a workflow configured and team repo has deployment type as workflow. - """ - filtered_team_repos: List[ - TeamRepos - ] = self._filter_team_repos_using_workflow_deployments(team_repos) - - repo_ids = [str(tr.org_repo_id) for tr in filtered_team_repos] - repo_id_to_team_repo_map = { - str(tr.org_repo_id): tr for tr in filtered_team_repos - } - - repo_workflows: List[ - RepoWorkflow - ] = self.workflow_repo_service.get_repo_workflow_by_repo_ids( - repo_ids, RepoWorkflowType.DEPLOYMENT - ) - workflows_repo_ids = list( - set([str(workflow.org_repo_id) for workflow in repo_workflows]) - ) - - team_repos_with_workflow_deployments = [ - repo_id_to_team_repo_map[repo_id] - for repo_id in workflows_repo_ids - if repo_id in repo_id_to_team_repo_map - ] - - return team_repos_with_workflow_deployments - - def get_team_all_deployments_in_interval( - self, - team_id: str, - interval, - pr_filter: PRFilter = None, - workflow_filter: WorkflowFilter = None, - ) -> List[Deployment]: - - team_repos = self._get_team_repos_by_team_id(team_id) - ( - team_repos_using_workflow_deployments, - team_repos_using_pr_deployments, - ) = self.get_filtered_team_repos_by_deployment_config(team_repos) - - deployments_using_workflow = self.workflow_based_deployments_service.get_repos_all_deployments_in_interval( - self._get_repo_ids_from_team_repos(team_repos_using_workflow_deployments), - interval, - workflow_filter, - ) - deployments_using_pr = ( - self.pr_based_deployments_service.get_repos_all_deployments_in_interval( - self._get_repo_ids_from_team_repos(team_repos_using_pr_deployments), - interval, - pr_filter, - ) - ) - - deployments: List[Deployment] = ( - deployments_using_workflow + deployments_using_pr - ) - sorted_deployments = self._sort_deployments_by_date(deployments) - - return sorted_deployments - - def _get_team_repos_by_team_id(self, team_id: str) -> List[TeamRepos]: - return self.code_repo_service.get_active_team_repos_by_team_id(team_id) - - def _get_repo_ids_from_team_repos(self, team_repos: List[TeamRepos]) -> List[str]: - return [str(team_repo.org_repo_id) for team_repo in team_repos] - - def get_filtered_team_repos_by_deployment_config( - self, team_repos: List[TeamRepos] - ) -> Tuple[List[TeamRepos], List[TeamRepos]]: - """ - Splits the input TeamRepos list into two TeamRepos List, TeamRepos using workflow and TeamRepos using pr deployments. - """ - return self._filter_team_repos_using_workflow_deployments( - team_repos - ), self._filter_team_repos_using_pr_deployments(team_repos) - - def _filter_team_repos_using_workflow_deployments( - self, team_repos: List[TeamRepos] - ): - return [ - team_repo - for team_repo in team_repos - if team_repo.deployment_type.value == DeploymentType.WORKFLOW.value - ] - - def _filter_team_repos_using_pr_deployments(self, team_repos: List[TeamRepos]): - return [ - team_repo - for team_repo in team_repos - if team_repo.deployment_type.value == DeploymentType.PR_MERGE.value - ] - - def _sort_deployments_by_date( - self, deployments: List[Deployment] - ) -> List[Deployment]: - return sorted(deployments, key=lambda deployment: deployment.conducted_at) - - -def get_deployments_service() -> DeploymentsService: - return DeploymentsService( - CodeRepoService(), - WorkflowRepoService(), - get_deployments_factory(DeploymentType.WORKFLOW), - get_deployments_factory(DeploymentType.PR_MERGE), - ) diff --git a/apiserver/dora/service/deployments/deployments_factory_service.py b/apiserver/dora/service/deployments/deployments_factory_service.py deleted file mode 100644 index 95d5316e8..000000000 --- a/apiserver/dora/service/deployments/deployments_factory_service.py +++ /dev/null @@ -1,46 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List, Dict, Tuple -from urllib.parse import unquote -from dora.store.models.code.pull_requests import PullRequest -from uuid import UUID -from werkzeug.exceptions import BadRequest - -from dora.store.models.code.repository import TeamRepos -from dora.service.deployments.models.models import Deployment, DeploymentType - - -class DeploymentsFactoryService(ABC): - @abstractmethod - def get_repos_successful_deployments_in_interval( - self, repo_ids, interval, specific_filter - ) -> List[Deployment]: - pass - - @abstractmethod - def get_repos_all_deployments_in_interval( - self, repo_ids, interval, specific_filter - ) -> List[Deployment]: - pass - - @abstractmethod - def get_pull_requests_related_to_deployment( - self, deployment: Deployment - ) -> List[PullRequest]: - pass - - @abstractmethod - def get_deployment_by_entity_id(self, entity_id: str) -> Deployment: - pass - - @classmethod - def get_deployment_type_and_entity_id_from_deployment_id( - cls, id_str: str - ) -> Tuple[DeploymentType, str]: - id_str = unquote(id_str) - # Split the id string by '|' - deployment_type, entity_id = id_str.split("|") - try: - UUID(entity_id) - except ValueError: - raise BadRequest(f"Invalid UUID entity id: {entity_id}") - return DeploymentType(deployment_type), entity_id diff --git a/apiserver/dora/service/deployments/factory.py b/apiserver/dora/service/deployments/factory.py deleted file mode 100644 index fba02ec27..000000000 --- a/apiserver/dora/service/deployments/factory.py +++ /dev/null @@ -1,27 +0,0 @@ -from .models.adapter import DeploymentsAdaptorFactory -from dora.service.deployments.models.models import DeploymentType -from dora.store.repos.code import CodeRepoService -from dora.store.repos.workflows import WorkflowRepoService -from .deployment_pr_mapper import DeploymentPRMapperService -from .deployments_factory_service import DeploymentsFactoryService -from .pr_deployments_service import PRDeploymentsService -from .workflow_deployments_service import WorkflowDeploymentsService - - -def get_deployments_factory( - deployment_type: DeploymentType, -) -> DeploymentsFactoryService: - if deployment_type == DeploymentType.PR_MERGE: - return PRDeploymentsService( - CodeRepoService(), - DeploymentsAdaptorFactory(DeploymentType.PR_MERGE).get_adaptor(), - ) - elif deployment_type == DeploymentType.WORKFLOW: - return WorkflowDeploymentsService( - WorkflowRepoService(), - CodeRepoService(), - DeploymentsAdaptorFactory(DeploymentType.WORKFLOW).get_adaptor(), - DeploymentPRMapperService(), - ) - else: - raise ValueError(f"Unknown deployment type: {deployment_type}") diff --git a/apiserver/dora/service/deployments/models/__init__.py b/apiserver/dora/service/deployments/models/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/dora/service/deployments/models/adapter.py b/apiserver/dora/service/deployments/models/adapter.py deleted file mode 100644 index 17397bf04..000000000 --- a/apiserver/dora/service/deployments/models/adapter.py +++ /dev/null @@ -1,105 +0,0 @@ -from abc import ABC -from typing import Union, List, Tuple -from dora.store.models.code.enums import PullRequestState -from dora.store.models.code.pull_requests import PullRequest - -from dora.store.models.code.workflows.workflows import RepoWorkflow, RepoWorkflowRuns -from dora.service.deployments.models.models import ( - Deployment, - DeploymentStatus, - DeploymentType, -) - - -class DeploymentsAdaptor(ABC): - def adapt(self, entity: Union[Tuple[RepoWorkflow, RepoWorkflowRuns], PullRequest]): - pass - - def adapt_many( - self, entities: List[Union[Tuple[RepoWorkflow, RepoWorkflowRuns], PullRequest]] - ): - pass - - -class DeploymentsAdaptorFactory: - def __init__(self, deployment_type: DeploymentType): - self.deployment_type = deployment_type - - def get_adaptor(self) -> DeploymentsAdaptor: - if self.deployment_type == DeploymentType.WORKFLOW: - return WorkflowRunsToDeploymentsAdaptor() - elif self.deployment_type == DeploymentType.PR_MERGE: - return PullRequestToDeploymentsAdaptor() - else: - raise ValueError( - f"Unsupported deployment type: {self.deployment_type.value}" - ) - - -class WorkflowRunsToDeploymentsAdaptor(DeploymentsAdaptor): - def adapt(self, entity: Tuple[RepoWorkflow, RepoWorkflowRuns]): - repo_workflow, repo_workflow_run = entity - return Deployment( - deployment_type=DeploymentType.WORKFLOW, - repo_id=str(repo_workflow.org_repo_id), - entity_id=str(repo_workflow_run.id), - provider=repo_workflow.provider.value, - actor=repo_workflow_run.event_actor, - head_branch=repo_workflow_run.head_branch, - conducted_at=repo_workflow_run.conducted_at, - duration=repo_workflow_run.duration, - status=DeploymentStatus(repo_workflow_run.status.value), - html_url=repo_workflow_run.html_url, - meta=dict( - id=str(repo_workflow.id), - repo_workflow_id=str(repo_workflow_run.repo_workflow_id), - provider_workflow_run_id=repo_workflow_run.provider_workflow_run_id, - event_actor=repo_workflow_run.event_actor, - head_branch=repo_workflow_run.head_branch, - status=repo_workflow_run.status.value, - conducted_at=repo_workflow_run.conducted_at.isoformat(), - duration=repo_workflow_run.duration, - html_url=repo_workflow_run.html_url, - ), - ) - - def adapt_many(self, entities: List[Tuple[RepoWorkflow, RepoWorkflowRuns]]): - return [self.adapt(entity) for entity in entities] - - -class PullRequestToDeploymentsAdaptor(DeploymentsAdaptor): - def adapt(self, entity: PullRequest): - if not self._is_pull_request_merged(entity): - raise ValueError("Pull request is not merged") - return Deployment( - deployment_type=DeploymentType.PR_MERGE, - repo_id=str(entity.repo_id), - entity_id=str(entity.id), - provider=entity.provider, - actor=entity.username, - head_branch=entity.base_branch, - conducted_at=entity.state_changed_at, - duration=0, - status=DeploymentStatus.SUCCESS, - html_url=entity.url, - meta=dict( - id=str(entity.id), - repo_id=str(entity.repo_id), - number=entity.number, - provider=entity.provider, - username=entity.username, - base_branch=entity.base_branch, - state_changed_at=entity.state_changed_at.isoformat(), - url=entity.url, - ), - ) - - def adapt_many(self, entities: List[PullRequest]): - return [ - self.adapt(entity) - for entity in entities - if self._is_pull_request_merged(entity) - ] - - def _is_pull_request_merged(self, entity: PullRequest): - return entity.state == PullRequestState.MERGED diff --git a/apiserver/dora/service/deployments/models/models.py b/apiserver/dora/service/deployments/models/models.py deleted file mode 100644 index d3a4676f1..000000000 --- a/apiserver/dora/service/deployments/models/models.py +++ /dev/null @@ -1,46 +0,0 @@ -from dataclasses import dataclass -from datetime import datetime -from enum import Enum -from voluptuous import default_factory - - -class DeploymentType(Enum): - WORKFLOW = "WORKFLOW" - PR_MERGE = "PR_MERGE" - - -class DeploymentStatus(Enum): - SUCCESS = "SUCCESS" - FAILURE = "FAILURE" - PENDING = "PENDING" - CANCELLED = "CANCELLED" - - -@dataclass -class Deployment: - deployment_type: DeploymentType - repo_id: str - entity_id: str - provider: str - actor: str - head_branch: str - conducted_at: datetime - duration: int - status: DeploymentStatus - html_url: str - meta: dict = default_factory(dict) - - def __hash__(self): - return hash(self.deployment_type.value + "|" + str(self.entity_id)) - - @property - def id(self): - return self.deployment_type.value + "|" + str(self.entity_id) - - -@dataclass -class DeploymentFrequencyMetrics: - total_deployments: int - daily_deployment_frequency: int - avg_weekly_deployment_frequency: int - avg_monthly_deployment_frequency: int diff --git a/apiserver/dora/service/deployments/pr_deployments_service.py b/apiserver/dora/service/deployments/pr_deployments_service.py deleted file mode 100644 index f3292b8a6..000000000 --- a/apiserver/dora/service/deployments/pr_deployments_service.py +++ /dev/null @@ -1,51 +0,0 @@ -from typing import List -from .models.adapter import DeploymentsAdaptor -from dora.store.models.code.filter import PRFilter -from dora.store.models.code.pull_requests import PullRequest -from dora.service.deployments.models.models import Deployment - -from dora.store.repos.code import CodeRepoService -from dora.utils.time import Interval - -from .deployments_factory_service import DeploymentsFactoryService - - -class PRDeploymentsService(DeploymentsFactoryService): - def __init__( - self, - code_repo_service: CodeRepoService, - deployments_adapter: DeploymentsAdaptor, - ): - self.code_repo_service = code_repo_service - self.deployments_adapter = deployments_adapter - - def get_repos_successful_deployments_in_interval( - self, repo_ids: List[str], interval: Interval, pr_filter: PRFilter - ) -> List[Deployment]: - pull_requests: List[ - PullRequest - ] = self.code_repo_service.get_prs_merged_in_interval( - repo_ids, interval, pr_filter=pr_filter - ) - - return self.deployments_adapter.adapt_many(pull_requests) - - def get_repos_all_deployments_in_interval( - self, repo_ids: List[str], interval: Interval, prs_filter: PRFilter - ) -> List[Deployment]: - return self.get_repos_successful_deployments_in_interval( - repo_ids, interval, prs_filter - ) - - def get_pull_requests_related_to_deployment( - self, deployment: Deployment - ) -> List[PullRequest]: - return [self.code_repo_service.get_pull_request_by_id(deployment.entity_id)] - - def get_deployment_by_entity_id(self, entity_id: str) -> Deployment: - pull_request: PullRequest = self.code_repo_service.get_pull_request_by_id( - entity_id - ) - if not pull_request: - raise ValueError(f"Pull Request with id {entity_id} not found") - return self.deployments_adapter.adapt(pull_request) diff --git a/apiserver/dora/service/deployments/workflow_deployments_service.py b/apiserver/dora/service/deployments/workflow_deployments_service.py deleted file mode 100644 index 2d91258cc..000000000 --- a/apiserver/dora/service/deployments/workflow_deployments_service.py +++ /dev/null @@ -1,92 +0,0 @@ -from typing import List, Tuple -from .models.adapter import DeploymentsAdaptor -from dora.store.models.code.pull_requests import PullRequest -from dora.store.models.code.repository import TeamRepos -from dora.store.models.code.workflows.filter import WorkflowFilter -from dora.store.models.code.workflows.workflows import RepoWorkflow, RepoWorkflowRuns -from dora.service.deployments.models.models import Deployment -from dora.store.repos.code import CodeRepoService - -from dora.store.repos.workflows import WorkflowRepoService -from dora.utils.time import Interval - -from .deployment_pr_mapper import DeploymentPRMapperService -from .deployments_factory_service import DeploymentsFactoryService - - -class WorkflowDeploymentsService(DeploymentsFactoryService): - def __init__( - self, - workflow_repo_service: WorkflowRepoService, - code_repo_service: CodeRepoService, - deployments_adapter: DeploymentsAdaptor, - deployment_pr_mapping_service: DeploymentPRMapperService, - ): - self.workflow_repo_service = workflow_repo_service - self.code_repo_service = code_repo_service - self.deployments_adapter = deployments_adapter - self.deployment_pr_mapping_service = deployment_pr_mapping_service - - def get_repos_successful_deployments_in_interval( - self, repo_ids: List[str], interval: Interval, workflow_filter: WorkflowFilter - ) -> List[Deployment]: - repo_workflow_runs: List[ - Tuple[RepoWorkflow, RepoWorkflowRuns] - ] = self.workflow_repo_service.get_successful_repo_workflows_runs_by_repo_ids( - repo_ids, interval, workflow_filter - ) - return self.deployments_adapter.adapt_many(repo_workflow_runs) - - def get_repos_all_deployments_in_interval( - self, - repo_ids: List[str], - interval: Interval, - workflow_filter: WorkflowFilter, - ) -> List[Deployment]: - repo_workflow_runs: List[ - Tuple[RepoWorkflow, RepoWorkflowRuns] - ] = self.workflow_repo_service.get_repos_workflow_runs_by_repo_ids( - repo_ids, interval, workflow_filter - ) - return self.deployments_adapter.adapt_many(repo_workflow_runs) - - def get_pull_requests_related_to_deployment( - self, deployment: Deployment - ) -> List[PullRequest]: - previous_deployment = self._get_previous_deployment_for_given_deployment( - deployment - ) - interval = Interval(previous_deployment.conducted_at, deployment.conducted_at) - pr_base_branch: str = deployment.head_branch - pull_requests: List[ - PullRequest - ] = self.code_repo_service.get_prs_merged_in_interval( - [deployment.repo_id], interval, base_branches=[pr_base_branch] - ) - relevant_prs: List[ - PullRequest - ] = self.deployment_pr_mapping_service.get_all_prs_deployed( - pull_requests, deployment - ) - - return relevant_prs - - def get_deployment_by_entity_id(self, entity_id: str) -> Deployment: - repo_workflow_run: Tuple[ - RepoWorkflow, RepoWorkflowRuns - ] = self.workflow_repo_service.get_repo_workflow_run_by_id(entity_id) - if not repo_workflow_run: - raise ValueError(f"Workflow run with id {entity_id} not found") - return self.deployments_adapter.adapt(repo_workflow_run) - - def _get_previous_deployment_for_given_deployment( - self, deployment: Deployment - ) -> Deployment: - ( - workflow_run, - current_workflow_run, - ) = self.workflow_repo_service.get_repo_workflow_run_by_id(deployment.entity_id) - workflow_run_previous_workflow_run: Tuple[ - RepoWorkflow, RepoWorkflowRuns - ] = self.workflow_repo_service.get_previous_workflow_run(current_workflow_run) - return self.deployments_adapter.adapt(workflow_run_previous_workflow_run) diff --git a/apiserver/dora/service/external_integrations_service.py b/apiserver/dora/service/external_integrations_service.py deleted file mode 100644 index 38e8c2fab..000000000 --- a/apiserver/dora/service/external_integrations_service.py +++ /dev/null @@ -1,63 +0,0 @@ -from github.Organization import Organization as GithubOrganization - -from dora.exapi.github import GithubApiService, GithubRateLimitExceeded -from dora.store.models import UserIdentityProvider -from dora.store.repos.core import CoreRepoService - -PAGE_SIZE = 100 - - -class ExternalIntegrationsService: - def __init__( - self, - org_id: str, - user_identity_provider: UserIdentityProvider, - access_token: str, - ): - self.org_id = org_id - self.user_identity_provider = user_identity_provider - self.access_token = access_token - - def get_github_organizations(self): - github_api_service = GithubApiService(self.access_token) - try: - orgs: [GithubOrganization] = github_api_service.get_org_list() - except GithubRateLimitExceeded as e: - raise Exception(e) - return orgs - - def get_github_org_repos(self, org_login: str, page_size: int, page: int): - github_api_service = GithubApiService(self.access_token) - try: - return github_api_service.get_repos_raw(org_login, page_size, page) - except Exception as e: - raise Exception(e) - - def get_repo_workflows(self, gh_org_name: str, gh_org_repo_name: str): - github_api_service = GithubApiService(self.access_token) - workflows = github_api_service.get_repo_workflows(gh_org_name, gh_org_repo_name) - workflows_list = [] - for page in range(0, workflows.totalCount // PAGE_SIZE + 1, 1): - workflows = workflows.get_page(page) - if not workflows: - break - workflows_list += workflows - return workflows_list - - -def get_external_integrations_service( - org_id: str, user_identity_provider: UserIdentityProvider -): - def _get_access_token() -> str: - access_token = CoreRepoService().get_access_token( - org_id, user_identity_provider - ) - if not access_token: - raise Exception( - f"Access token not found for org {org_id} and provider {user_identity_provider.value}" - ) - return access_token - - return ExternalIntegrationsService( - org_id, user_identity_provider, _get_access_token() - ) diff --git a/apiserver/dora/service/incidents/__init__.py b/apiserver/dora/service/incidents/__init__.py deleted file mode 100644 index 242f1f6fb..000000000 --- a/apiserver/dora/service/incidents/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .sync import sync_org_incidents diff --git a/apiserver/dora/service/incidents/incident_filter.py b/apiserver/dora/service/incidents/incident_filter.py deleted file mode 100644 index 43cfd7b3f..000000000 --- a/apiserver/dora/service/incidents/incident_filter.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Dict, List, Any, Optional -from dora.store.models.settings.configuration_settings import SettingType -from dora.service.settings.configuration_settings import ( - get_settings_service, - IncidentSettings, - IncidentTypesSetting, -) -from dora.store.models.incidents import IncidentFilter - -from dora.store.models.settings import EntityType - - -class IncidentFilterService: - def __init__( - self, - raw_incident_filter: Dict = None, - entity_type: EntityType = None, - entity_id: str = None, - setting_types: List[SettingType] = None, - setting_type_to_settings_map: Dict[SettingType, Any] = None, - ): - self.raw_incident_filter: Dict = raw_incident_filter or {} - self.entity_type: EntityType = entity_type - self.entity_id = entity_id - self.setting_types: List[SettingType] = setting_types or [] - self.setting_type_to_settings_map: Dict[SettingType, any] = ( - setting_type_to_settings_map or {} - ) - - def apply(self): - incident_filter: IncidentFilter = IncidentFilter() - if self.entity_type and self.entity_id: - incident_filter = ConfigurationsIncidentFilterProcessor( - incident_filter, - self.entity_type, - self.entity_id, - self.setting_types, - self.setting_type_to_settings_map, - ).apply() - return incident_filter - - -def apply_incident_filter( - incident_filter: Dict = None, - entity_type: EntityType = None, - entity_id: str = None, - setting_types: List[SettingType] = None, -) -> IncidentFilter: - setting_service = get_settings_service() - setting_type_to_settings_map = setting_service.get_settings_map( - entity_id, setting_types, entity_type - ) - - return IncidentFilterService( - incident_filter, - entity_type, - entity_id, - setting_types, - setting_type_to_settings_map, - ).apply() - - -class ConfigurationsIncidentFilterProcessor: - def __init__( - self, - incident_filter: IncidentFilter, - entity_type: EntityType, - entity_id: str, - setting_types: List[SettingType], - setting_type_to_settings_map: Dict[SettingType, Any], - ): - self.incident_filter = incident_filter or IncidentFilter() - self.entity_type: EntityType = entity_type - self.entity_id = entity_id - self.setting_types: List[SettingType] = setting_types or [] - self.setting_type_to_settings_map = setting_type_to_settings_map - - def apply(self): - if SettingType.INCIDENT_SETTING in self.setting_types: - self.incident_filter.title_filter_substrings = ( - self.__incident_title_filter() - ) - - if SettingType.INCIDENT_TYPES_SETTING in self.setting_types: - self.incident_filter.incident_types = self.__incident_type_setting() - - return self.incident_filter - - def __incident_title_filter(self) -> List[str]: - setting: Optional[IncidentSettings] = self.setting_type_to_settings_map.get( - SettingType.INCIDENT_SETTING - ) - if not setting: - return [] - title_filters = [] - if setting and isinstance(setting, IncidentSettings): - title_filters = setting.title_filters - return title_filters - - def __incident_type_setting(self) -> List[str]: - setting: Optional[IncidentTypesSetting] = self.setting_type_to_settings_map.get( - SettingType.INCIDENT_TYPES_SETTING - ) - if not setting: - return [] - incident_types = [] - if setting and isinstance(setting, IncidentTypesSetting): - incident_types = setting.incident_types - return incident_types diff --git a/apiserver/dora/service/incidents/incidents.py b/apiserver/dora/service/incidents/incidents.py deleted file mode 100644 index 6a3ccc1d8..000000000 --- a/apiserver/dora/service/incidents/incidents.py +++ /dev/null @@ -1,213 +0,0 @@ -from collections import defaultdict -from datetime import datetime -from typing import List, Dict, Tuple -from dora.service.incidents.models.mean_time_to_recovery import ( - ChangeFailureRateMetrics, - MeanTimeToRecoveryMetrics, -) -from dora.service.deployments.models.models import Deployment -from dora.service.incidents.incident_filter import apply_incident_filter -from dora.store.models.incidents.filter import IncidentFilter -from dora.store.models.settings import EntityType, SettingType -from dora.utils.time import ( - Interval, - fill_missing_week_buckets, - generate_expanded_buckets, - get_given_weeks_monday, -) - -from dora.store.models.incidents import Incident -from dora.service.settings.configuration_settings import ( - SettingsService, - get_settings_service, -) -from dora.store.repos.incidents import IncidentsRepoService - - -class IncidentService: - def __init__( - self, - incidents_repo_service: IncidentsRepoService, - settings_service: SettingsService, - ): - self._incidents_repo_service = incidents_repo_service - self._settings_service = settings_service - - def get_resolved_team_incidents( - self, team_id: str, interval: Interval - ) -> List[Incident]: - incident_filter: IncidentFilter = apply_incident_filter( - entity_type=EntityType.TEAM, - entity_id=team_id, - setting_types=[ - SettingType.INCIDENT_SETTING, - SettingType.INCIDENT_TYPES_SETTING, - ], - ) - return self._incidents_repo_service.get_resolved_team_incidents( - team_id, interval, incident_filter - ) - - def get_team_incidents(self, team_id: str, interval: Interval) -> List[Incident]: - incident_filter: IncidentFilter = apply_incident_filter( - entity_type=EntityType.TEAM, - entity_id=team_id, - setting_types=[ - SettingType.INCIDENT_SETTING, - SettingType.INCIDENT_TYPES_SETTING, - ], - ) - return self._incidents_repo_service.get_team_incidents( - team_id, interval, incident_filter - ) - - def get_deployment_incidents_map( - self, deployments: List[Deployment], incidents: List[Incident] - ): - deployments = sorted(deployments, key=lambda x: x.conducted_at) - incidents = sorted(incidents, key=lambda x: x.creation_date) - incidents_pointer = 0 - - deployment_incidents_map: Dict[Deployment, List[Incident]] = defaultdict(list) - - for current_deployment, next_deployment in zip( - deployments, deployments[1:] + [None] - ): - current_deployment_incidents = [] - - if incidents_pointer >= len(incidents): - deployment_incidents_map[ - current_deployment - ] = current_deployment_incidents - continue - - while incidents_pointer < len(incidents): - incident = incidents[incidents_pointer] - - if incident.creation_date >= current_deployment.conducted_at and ( - next_deployment is None - or incident.creation_date < next_deployment.conducted_at - ): - current_deployment_incidents.append(incident) - incidents_pointer += 1 - elif incident.creation_date < current_deployment.conducted_at: - incidents_pointer += 1 - else: - break - - deployment_incidents_map[current_deployment] = current_deployment_incidents - - return deployment_incidents_map - - def get_team_mean_time_to_recovery( - self, team_id: str, interval: Interval - ) -> MeanTimeToRecoveryMetrics: - - resolved_team_incidents = self.get_resolved_team_incidents(team_id, interval) - - return self._get_incidents_mean_time_to_recovery(resolved_team_incidents) - - def get_team_mean_time_to_recovery_trends( - self, team_id: str, interval: Interval - ) -> MeanTimeToRecoveryMetrics: - - resolved_team_incidents = self.get_resolved_team_incidents(team_id, interval) - - weekly_resolved_team_incidents: Dict[ - datetime, List[Incident] - ] = generate_expanded_buckets( - resolved_team_incidents, interval, "resolved_date", "weekly" - ) - - weekly_mean_time_to_recovery: Dict[datetime, MeanTimeToRecoveryMetrics] = {} - - for week, incidents in weekly_resolved_team_incidents.items(): - - if incidents: - weekly_mean_time_to_recovery[ - week - ] = self._get_incidents_mean_time_to_recovery(incidents) - else: - weekly_mean_time_to_recovery[week] = MeanTimeToRecoveryMetrics() - - return weekly_mean_time_to_recovery - - def calculate_change_failure_deployments( - self, deployment_incidents_map: Dict[Deployment, List[Incident]] - ) -> Tuple[List[Deployment], List[Deployment]]: - failed_deployments = [ - deployment - for deployment, incidents in deployment_incidents_map.items() - if incidents - ] - all_deployments: List[Deployment] = list(deployment_incidents_map.keys()) - - return failed_deployments, all_deployments - - def get_change_failure_rate_metrics( - self, deployments: List[Deployment], incidents: List[Incident] - ) -> ChangeFailureRateMetrics: - deployment_incidents_map = self.get_deployment_incidents_map( - deployments, incidents - ) - ( - failed_deployments, - all_deployments, - ) = self.calculate_change_failure_deployments(deployment_incidents_map) - return ChangeFailureRateMetrics(set(failed_deployments), set(all_deployments)) - - def get_weekly_change_failure_rate( - self, - interval: Interval, - deployments: List[Deployment], - incidents: List[Incident], - ) -> ChangeFailureRateMetrics: - - deployments_incidents_map = self.get_deployment_incidents_map( - deployments, incidents - ) - week_start_to_change_failure_rate_map: Dict[ - datetime, ChangeFailureRateMetrics - ] = defaultdict(ChangeFailureRateMetrics) - - for deployment, incidents in deployments_incidents_map.items(): - week_start_date = get_given_weeks_monday(deployment.conducted_at) - if incidents: - week_start_to_change_failure_rate_map[ - week_start_date - ].failed_deployments.add(deployment) - week_start_to_change_failure_rate_map[ - week_start_date - ].total_deployments.add(deployment) - - return fill_missing_week_buckets( - week_start_to_change_failure_rate_map, interval, ChangeFailureRateMetrics - ) - - def _calculate_incident_resolution_time(self, incident: Incident) -> int: - return (incident.resolved_date - incident.creation_date).total_seconds() - - def _get_incidents_mean_time_to_recovery( - self, resolved_incidents: List[Incident] - ) -> MeanTimeToRecoveryMetrics: - - incident_count = len(resolved_incidents) - - if not incident_count: - return MeanTimeToRecoveryMetrics() - - mean_time_to_recovery = ( - sum( - [ - self._calculate_incident_resolution_time(incident) - for incident in resolved_incidents - ] - ) - / incident_count - ) - - return MeanTimeToRecoveryMetrics(mean_time_to_recovery, incident_count) - - -def get_incident_service(): - return IncidentService(IncidentsRepoService(), get_settings_service()) diff --git a/apiserver/dora/service/incidents/integration.py b/apiserver/dora/service/incidents/integration.py deleted file mode 100644 index 5e3632896..000000000 --- a/apiserver/dora/service/incidents/integration.py +++ /dev/null @@ -1,65 +0,0 @@ -from typing import List - -from dora.service.settings import SettingsService, get_settings_service -from dora.service.settings.models import IncidentSourcesSetting -from dora.store.models import Integration, SettingType, EntityType -from dora.store.models.incidents import IncidentProvider, IncidentSource -from dora.store.repos.core import CoreRepoService - -GIT_INCIDENT_INTEGRATION_BUCKET = [IncidentProvider.GITHUB.value] - - -class IncidentsIntegrationService: - def __init__( - self, core_repo_service: CoreRepoService, settings_service: SettingsService - ): - self.core_repo_service = core_repo_service - self.settings_service = settings_service - - def get_org_providers(self, org_id: str) -> List[str]: - integrations: List[ - Integration - ] = self.core_repo_service.get_org_integrations_for_names( - org_id, self._get_possible_incident_providers(org_id) - ) - if not integrations: - return [] - return [integration.name for integration in integrations] - - def _get_possible_incident_providers(self, org_id: str) -> List[str]: - - valid_integration_types = [] - - incident_source_setting: IncidentSourcesSetting = ( - self._get_or_create_incident_source_setting(org_id) - ) - - if IncidentSource.GIT_REPO in incident_source_setting.incident_sources: - valid_integration_types += GIT_INCIDENT_INTEGRATION_BUCKET - - return valid_integration_types - - def _get_or_create_incident_source_setting( - self, org_id: str - ) -> IncidentSourcesSetting: - - settings = self.settings_service.get_settings( - setting_type=SettingType.INCIDENT_SOURCES_SETTING, - entity_type=EntityType.ORG, - entity_id=org_id, - ) - - if not settings: - settings = self.settings_service.save_settings( - setting_type=SettingType.INCIDENT_SOURCES_SETTING, - entity_type=EntityType.ORG, - entity_id=org_id, - ) - return settings.specific_settings - - -def get_incidents_integration_service(): - return IncidentsIntegrationService( - core_repo_service=CoreRepoService(), - settings_service=get_settings_service(), - ) diff --git a/apiserver/dora/service/incidents/models/mean_time_to_recovery.py b/apiserver/dora/service/incidents/models/mean_time_to_recovery.py deleted file mode 100644 index 509bf2447..000000000 --- a/apiserver/dora/service/incidents/models/mean_time_to_recovery.py +++ /dev/null @@ -1,34 +0,0 @@ -from dataclasses import dataclass -from typing import Optional, Set - -from dora.service.deployments.models.models import Deployment - - -@dataclass -class MeanTimeToRecoveryMetrics: - mean_time_to_recovery: Optional[float] = None - incident_count: int = 0 - - -@dataclass -class ChangeFailureRateMetrics: - failed_deployments: Set[Deployment] = None - total_deployments: Set[Deployment] = None - - def __post_init__(self): - self.failed_deployments = self.failed_deployments or set() - self.total_deployments = self.total_deployments or set() - - @property - def change_failure_rate(self): - if not self.total_deployments: - return 0 - return len(self.failed_deployments) / len(self.total_deployments) * 100 - - @property - def failed_deployments_count(self): - return len(self.failed_deployments) - - @property - def total_deployments_count(self): - return len(self.total_deployments) diff --git a/apiserver/dora/service/incidents/sync/__init__.py b/apiserver/dora/service/incidents/sync/__init__.py deleted file mode 100644 index 445ee7725..000000000 --- a/apiserver/dora/service/incidents/sync/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .etl_handler import sync_org_incidents diff --git a/apiserver/dora/service/incidents/sync/etl_git_incidents_handler.py b/apiserver/dora/service/incidents/sync/etl_git_incidents_handler.py deleted file mode 100644 index ee01914ea..000000000 --- a/apiserver/dora/service/incidents/sync/etl_git_incidents_handler.py +++ /dev/null @@ -1,242 +0,0 @@ -from datetime import datetime -from typing import List, Dict, Optional, Tuple - -from dora.exapi.git_incidents import ( - GitIncidentsAPIService, - get_git_incidents_api_service, -) -from dora.exapi.models.git_incidents import RevertPRMap -from dora.service.incidents.sync.etl_provider_handler import IncidentsProviderETLHandler -from dora.store.models.code import OrgRepo, PullRequest -from dora.store.models.incidents import ( - IncidentSource, - OrgIncidentService, - IncidentType, - IncidentsBookmark, - IncidentOrgIncidentServiceMap, - IncidentStatus, - Incident, - IncidentProvider, -) -from dora.store.repos.incidents import IncidentsRepoService -from dora.utils.log import LOG -from dora.utils.string import uuid4_str -from dora.utils.time import time_now - - -class GitIncidentsETLHandler(IncidentsProviderETLHandler): - def __init__( - self, - org_id: str, - git_incidents_api_service: GitIncidentsAPIService, - incidents_repo_service: IncidentsRepoService, - ): - self.org_id = org_id - self.git_incidents_api_service = git_incidents_api_service - self.incidents_repo_service = incidents_repo_service - - def check_pat_validity(self) -> bool: - """ - Checks if Incident Source, "GIT_REPO" is enabled for the org - :return: True if enabled, False otherwise - """ - return self.git_incidents_api_service.is_sync_enabled(self.org_id) - - def get_updated_incident_services( - self, incident_services: List[OrgIncidentService] - ) -> List[OrgIncidentService]: - """ - Get the updated Incident Services for the org - :param incident_services: List of Incident Services - :return: List of updated Incident Services - """ - git_repo_type_incident_services = [ - incident_service - for incident_service in incident_services - if incident_service.source_type == IncidentSource.GIT_REPO - ] - active_org_repos: List[OrgRepo] = self.git_incidents_api_service.get_org_repos( - self.org_id - ) - - key_to_service_map: Dict[str, OrgIncidentService] = { - incident_service.key: incident_service - for incident_service in git_repo_type_incident_services - } - - updated_services: List[OrgIncidentService] = [] - - for org_repo in active_org_repos: - updated_services.append( - self._adapt_org_incident_service( - org_repo, key_to_service_map.get(str(org_repo.id)) - ) - ) - - return updated_services - - def process_service_incidents( - self, - incident_service: OrgIncidentService, - bookmark: IncidentsBookmark, - ) -> Tuple[List[Incident], List[IncidentOrgIncidentServiceMap], IncidentsBookmark]: - """ - Sync incidents for the service - :param incident_service: OrgIncidentService - :param bookmark: IncidentsBookmark - :return: List of Incidents, List of IncidentOrgIncidentServiceMap, IncidentsBookmark - """ - if not incident_service or not isinstance(incident_service, OrgIncidentService): - raise Exception(f"Service not found") - - from_time: datetime = bookmark.bookmark - to_time: datetime = time_now() - - revert_pr_incidents: List[ - RevertPRMap - ] = self.git_incidents_api_service.get_repo_revert_prs_in_interval( - incident_service.key, from_time, to_time - ) - if not revert_pr_incidents: - LOG.warning( - f"[GIT Incidents Sync] Incidents not received for service {str(incident_service.id)} " - f"in org {self.org_id} since {from_time.isoformat()}" - ) - return [], [], bookmark - - revert_pr_incidents.sort( - key=lambda revert_pr_incident: revert_pr_incident.updated_at - ) - - bookmark.bookmark = max(bookmark.bookmark, revert_pr_incidents[-1].updated_at) - - incidents, incident_org_incident_service_map_models = self._process_incidents( - incident_service, revert_pr_incidents - ) - - return incidents, incident_org_incident_service_map_models, bookmark - - def _process_incidents( - self, - org_incident_service: OrgIncidentService, - revert_pr_incidents: List[RevertPRMap], - ) -> Tuple[List[Incident], List[IncidentOrgIncidentServiceMap]]: - - if not revert_pr_incidents: - LOG.warning( - f"[GitIncidentsService Incident Sync] Incidents not received for " - f"service {str(org_incident_service.id)} in org {self.org_id}" - ) - return [], [] - - incident_models = [] - incident_org_incident_service_map_models = [] - - for revert_pr_incident in revert_pr_incidents: - try: - incident, incident_service_map = self._process_revert_pr_incident( - org_incident_service, revert_pr_incident - ) - incident_models.append(incident) - incident_org_incident_service_map_models.append(incident_service_map) - except Exception as e: - LOG.error( - f"ERROR processing revert pr Incident in service {str(org_incident_service.id)} in " - f"org {str(org_incident_service.org_id)}, Error: {str(e)}" - ) - raise e - - return incident_models, incident_org_incident_service_map_models - - def _process_revert_pr_incident( - self, org_incident_service: OrgIncidentService, revert_pr_map: RevertPRMap - ) -> Tuple[Incident, IncidentOrgIncidentServiceMap]: - incident_unique_id = str(revert_pr_map.original_pr.id) - existing_incident: Optional[ - Incident - ] = self.incidents_repo_service.get_incident_by_key_type_and_provider( - incident_unique_id, - IncidentType.REVERT_PR, - IncidentProvider(org_incident_service.provider), - ) - incident_id = existing_incident.id if existing_incident else uuid4_str() - - incident = Incident( - id=incident_id, - provider=org_incident_service.provider, - key=str(incident_unique_id), - title=revert_pr_map.original_pr.title, - incident_number=int(revert_pr_map.original_pr.number), - status=IncidentStatus.RESOLVED.value, - creation_date=revert_pr_map.original_pr.state_changed_at, - acknowledged_date=revert_pr_map.revert_pr.created_at, - resolved_date=revert_pr_map.revert_pr.state_changed_at, - assigned_to=revert_pr_map.revert_pr.author, - assignees=[revert_pr_map.revert_pr.author], - meta={ - "revert_pr": self._adapt_pr_to_json(revert_pr_map.revert_pr), - "original_pr": self._adapt_pr_to_json(revert_pr_map.original_pr), - "created_at": revert_pr_map.revert_pr.created_at.isoformat(), - "updated_at": revert_pr_map.revert_pr.updated_at.isoformat(), - }, - created_at=existing_incident.created_at - if existing_incident - else time_now(), - updated_at=time_now(), - incident_type=IncidentType.REVERT_PR, - ) - incident_org_incident_service_map_model = IncidentOrgIncidentServiceMap( - incident_id=incident_id, - service_id=org_incident_service.id, - ) - - return incident, incident_org_incident_service_map_model - - @staticmethod - def _adapt_org_incident_service( - org_repo: OrgRepo, - org_incident_service: OrgIncidentService, - ) -> OrgIncidentService: - - return OrgIncidentService( - id=org_incident_service.id if org_incident_service else uuid4_str(), - org_id=org_repo.org_id, - provider=org_repo.provider, - name=org_repo.name, - key=str(org_repo.id), - meta={}, - created_at=org_incident_service.created_at - if org_incident_service - else time_now(), - updated_at=time_now(), - source_type=IncidentSource.GIT_REPO, - ) - - @staticmethod - def _adapt_pr_to_json(pr: PullRequest) -> Dict[str, any]: - return { - "id": str(pr.id), - "repo_id": str(pr.repo_id), - "number": pr.number, - "title": pr.title, - "state": pr.state.value, - "author": pr.author, - "reviewers": pr.reviewers or [], - "url": pr.url, - "base_branch": pr.base_branch, - "head_branch": pr.head_branch, - "state_changed_at": pr.state_changed_at.isoformat() - if pr.state_changed_at - else None, - "commits": pr.commits, - "comments": pr.comments, - "provider": pr.provider, - } - - -def get_incidents_sync_etl_handler(org_id: str) -> GitIncidentsETLHandler: - return GitIncidentsETLHandler( - org_id, - get_git_incidents_api_service(), - IncidentsRepoService(), - ) diff --git a/apiserver/dora/service/incidents/sync/etl_handler.py b/apiserver/dora/service/incidents/sync/etl_handler.py deleted file mode 100644 index 455a48c84..000000000 --- a/apiserver/dora/service/incidents/sync/etl_handler.py +++ /dev/null @@ -1,107 +0,0 @@ -from datetime import timedelta -from typing import List - -from dora.service.incidents.integration import get_incidents_integration_service -from dora.service.incidents.sync.etl_incidents_factory import IncidentsETLFactory -from dora.service.incidents.sync.etl_provider_handler import IncidentsProviderETLHandler -from dora.store.models.incidents import ( - OrgIncidentService, - IncidentBookmarkType, - IncidentProvider, - IncidentsBookmark, -) -from dora.store.repos.incidents import IncidentsRepoService -from dora.utils.log import LOG -from dora.utils.string import uuid4_str -from dora.utils.time import time_now - - -class IncidentsETLHandler: - def __init__( - self, - provider: IncidentProvider, - incident_repo_service: IncidentsRepoService, - etl_service: IncidentsProviderETLHandler, - ): - self.provider = provider - self.incident_repo_service = incident_repo_service - self.etl_service = etl_service - - def sync_org_incident_services(self, org_id: str): - try: - incident_services = self.incident_repo_service.get_org_incident_services( - org_id - ) - updated_services = self.etl_service.get_updated_incident_services( - incident_services - ) - self.incident_repo_service.update_org_incident_services(updated_services) - for service in updated_services: - try: - self._sync_service_incidents(service) - except Exception as e: - LOG.error( - f"Error syncing incidents for service {service.key}: {str(e)}" - ) - continue - except Exception as e: - LOG.error(f"Error syncing incident services for org {org_id}: {str(e)}") - return - - def _sync_service_incidents(self, service: OrgIncidentService): - try: - bookmark = self.__get_incidents_bookmark(service) - ( - incidents, - incident_org_incident_service_map, - bookmark, - ) = self.etl_service.process_service_incidents(service, bookmark) - self.incident_repo_service.save_incidents_data( - incidents, incident_org_incident_service_map - ) - self.incident_repo_service.save_incidents_bookmark(bookmark) - - except Exception as e: - LOG.error(f"Error syncing incidents for service {service.key}: {str(e)}") - return - - def __get_incidents_bookmark( - self, service: OrgIncidentService, default_sync_days: int = 31 - ): - bookmark = self.incident_repo_service.get_incidents_bookmark( - str(service.id), IncidentBookmarkType.SERVICE, self.provider - ) - if not bookmark: - default_pr_bookmark = time_now() - timedelta(days=default_sync_days) - bookmark = IncidentsBookmark( - id=uuid4_str(), - entity_id=str(service.id), - entity_type=IncidentBookmarkType.SERVICE, - provider=self.provider.value, - bookmark=default_pr_bookmark, - ) - return bookmark - - -def sync_org_incidents(org_id: str): - incident_providers: List[ - str - ] = get_incidents_integration_service().get_org_providers(org_id) - if not incident_providers: - LOG.info(f"No incident providers found for org {org_id}") - return - etl_factory = IncidentsETLFactory(org_id) - - for provider in incident_providers: - try: - incident_provider = IncidentProvider(provider) - incidents_etl_handler = IncidentsETLHandler( - incident_provider, IncidentsRepoService(), etl_factory(provider) - ) - incidents_etl_handler.sync_org_incident_services(org_id) - except Exception as e: - LOG.error( - f"Error syncing incidents for provider {provider}, org {org_id}: {str(e)}" - ) - continue - LOG.info(f"Synced incidents for org {org_id}") diff --git a/apiserver/dora/service/incidents/sync/etl_incidents_factory.py b/apiserver/dora/service/incidents/sync/etl_incidents_factory.py deleted file mode 100644 index 1d536bf0b..000000000 --- a/apiserver/dora/service/incidents/sync/etl_incidents_factory.py +++ /dev/null @@ -1,15 +0,0 @@ -from dora.service.incidents.sync.etl_git_incidents_handler import ( - get_incidents_sync_etl_handler, -) -from dora.service.incidents.sync.etl_provider_handler import IncidentsProviderETLHandler -from dora.store.models.incidents import IncidentProvider - - -class IncidentsETLFactory: - def __init__(self, org_id: str): - self.org_id = org_id - - def __call__(self, provider: str) -> IncidentsProviderETLHandler: - if provider == IncidentProvider.GITHUB.value: - return get_incidents_sync_etl_handler(self.org_id) - raise NotImplementedError(f"Unknown provider - {provider}") diff --git a/apiserver/dora/service/incidents/sync/etl_provider_handler.py b/apiserver/dora/service/incidents/sync/etl_provider_handler.py deleted file mode 100644 index 9a8eeadae..000000000 --- a/apiserver/dora/service/incidents/sync/etl_provider_handler.py +++ /dev/null @@ -1,43 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List, Tuple - -from dora.store.models.incidents import ( - OrgIncidentService, - IncidentsBookmark, - Incident, - IncidentOrgIncidentServiceMap, -) - - -class IncidentsProviderETLHandler(ABC): - @abstractmethod - def check_pat_validity(self) -> bool: - """ - This method checks if the PAT is valid. - :return: True if PAT is valid, False otherwise - :raises: Exception if PAT is invalid - """ - pass - - @abstractmethod - def get_updated_incident_services( - self, incident_services: List[OrgIncidentService] - ) -> List[OrgIncidentService]: - """ - This method returns the updated incident services. - :param incident_services: List of incident services - :return: List of updated incident services - """ - pass - - @abstractmethod - def process_service_incidents( - self, incident_service: OrgIncidentService, bookmark: IncidentsBookmark - ) -> Tuple[List[Incident], List[IncidentOrgIncidentServiceMap], IncidentsBookmark]: - """ - This method processes the incidents for the incident services. - :param incident_service: Incident service object - :param bookmark: IncidentsBookmark object - :return: Tuple of incidents, incident service map and incidents bookmark - """ - pass diff --git a/apiserver/dora/service/merge_to_deploy_broker/__init__.py b/apiserver/dora/service/merge_to_deploy_broker/__init__.py deleted file mode 100644 index 9eca24263..000000000 --- a/apiserver/dora/service/merge_to_deploy_broker/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .mtd_handler import process_merge_to_deploy_cache -from .utils import get_merge_to_deploy_broker_utils_service, MergeToDeployBrokerUtils diff --git a/apiserver/dora/service/merge_to_deploy_broker/mtd_handler.py b/apiserver/dora/service/merge_to_deploy_broker/mtd_handler.py deleted file mode 100644 index 1196fe770..000000000 --- a/apiserver/dora/service/merge_to_deploy_broker/mtd_handler.py +++ /dev/null @@ -1,126 +0,0 @@ -from datetime import datetime -from typing import List - -from dora.service.deployments import DeploymentPRMapperService -from dora.store.models.code import ( - PullRequest, - OrgRepo, - RepoWorkflow, - BookmarkMergeToDeployBroker, - RepoWorkflowRuns, - RepoWorkflowRunsStatus, -) -from dora.store.repos.code import CodeRepoService -from dora.store.repos.workflows import WorkflowRepoService -from dora.utils.lock import RedisLockService, get_redis_lock_service - -DEPLOYMENTS_TO_PROCESS = 500 - - -class MergeToDeployCacheHandler: - def __init__( - self, - org_id: str, - code_repo_service: CodeRepoService, - workflow_repo_service: WorkflowRepoService, - deployment_pr_mapper_service: DeploymentPRMapperService, - redis_lock_service: RedisLockService, - ): - self.org_id = org_id - self.code_repo_service = code_repo_service - self.workflow_repo_service = workflow_repo_service - self.deployment_pr_mapper_service = deployment_pr_mapper_service - self.redis_lock_service = redis_lock_service - - def process_org_mtd(self): - org_repos: List[OrgRepo] = self.code_repo_service.get_active_org_repos( - self.org_id - ) - for org_repo in org_repos: - try: - with self.redis_lock_service.acquire_lock( - "{org_repo}:" + f"{str(org_repo.id)}:merge_to_deploy_broker" - ): - self._process_deployments_for_merge_to_deploy_caching( - str(org_repo.id) - ) - except Exception as e: - print(f"Error syncing workflow for repo {str(org_repo.id)}: {str(e)}") - continue - - def _process_deployments_for_merge_to_deploy_caching(self, repo_id: str): - org_repo: OrgRepo = self.code_repo_service.get_repo_by_id(repo_id) - if not org_repo: - Exception(f"Repo with {repo_id} not found") - - repo_workflows: List[ - RepoWorkflow - ] = self.workflow_repo_service.get_repo_workflows_by_repo_id(repo_id) - if not repo_workflows: - return - - broker_bookmark: BookmarkMergeToDeployBroker = ( - self.code_repo_service.get_merge_to_deploy_broker_bookmark(repo_id) - ) - if not broker_bookmark: - broker_bookmark = BookmarkMergeToDeployBroker(repo_id=repo_id) - - bookmark_time: datetime = broker_bookmark.bookmark_date - - repo_workflow_runs: List[ - RepoWorkflowRuns - ] = self.workflow_repo_service.get_repo_workflow_runs_conducted_after_time( - repo_id, bookmark_time, DEPLOYMENTS_TO_PROCESS - ) - - if not repo_workflow_runs: - return - - for repo_workflow_run in repo_workflow_runs: - try: - self.code_repo_service.get_merge_to_deploy_broker_bookmark(repo_id) - self._cache_prs_merge_to_deploy_for_repo_workflow_run( - repo_id, repo_workflow_run - ) - conducted_at: datetime = repo_workflow_run.conducted_at - broker_bookmark.bookmark = conducted_at.isoformat() - self.code_repo_service.update_merge_to_deploy_broker_bookmark( - broker_bookmark - ) - except Exception as e: - raise Exception(f"Error caching prs for repo {repo_id}: {str(e)}") - - def _cache_prs_merge_to_deploy_for_repo_workflow_run( - self, repo_id: str, repo_workflow_run: RepoWorkflowRuns - ): - if repo_workflow_run.status != RepoWorkflowRunsStatus.SUCCESS: - return - - conducted_at: datetime = repo_workflow_run.conducted_at - relevant_prs: List[ - PullRequest - ] = self.code_repo_service.get_prs_in_repo_merged_before_given_date_with_merge_to_deploy_as_null( - repo_id, conducted_at - ) - prs_to_update: List[ - PullRequest - ] = self.deployment_pr_mapper_service.get_all_prs_deployed( - relevant_prs, repo_workflow_run - ) - - for pr in prs_to_update: - pr.merge_to_deploy = int( - (conducted_at - pr.state_changed_at).total_seconds() - ) - self.code_repo_service.update_prs(prs_to_update) - - -def process_merge_to_deploy_cache(org_id: str): - merge_to_deploy_cache_handler = MergeToDeployCacheHandler( - org_id, - CodeRepoService(), - WorkflowRepoService(), - DeploymentPRMapperService(), - get_redis_lock_service(), - ) - merge_to_deploy_cache_handler.process_org_mtd() diff --git a/apiserver/dora/service/merge_to_deploy_broker/utils.py b/apiserver/dora/service/merge_to_deploy_broker/utils.py deleted file mode 100644 index eefdf3fee..000000000 --- a/apiserver/dora/service/merge_to_deploy_broker/utils.py +++ /dev/null @@ -1,49 +0,0 @@ -from datetime import datetime -from typing import List - -from dora.store.models.code import ( - PullRequest, - PullRequestState, - BookmarkMergeToDeployBroker, -) -from dora.store.repos.code import CodeRepoService -from dora.utils.lock import get_redis_lock_service, RedisLockService - - -class MergeToDeployBrokerUtils: - def __init__( - self, code_repo_service: CodeRepoService, redis_lock_service: RedisLockService - ): - self.code_repo_service = code_repo_service - self.redis_lock_service = redis_lock_service - - def pushback_merge_to_deploy_bookmark(self, repo_id: str, prs: List[PullRequest]): - with self.redis_lock_service.acquire_lock( - "{org_repo}:" + f"{repo_id}:merge_to_deploy_broker" - ): - self._pushback_merge_to_deploy_bookmark(repo_id, prs) - - def _pushback_merge_to_deploy_bookmark(self, repo_id: str, prs: List[PullRequest]): - merged_prs = [pr for pr in prs if pr.state == PullRequestState.MERGED] - if not merged_prs: - return - - min_merged_time: datetime = min([pr.state_changed_at for pr in merged_prs]) - - merge_to_deploy_broker_bookmark: BookmarkMergeToDeployBroker = ( - self.code_repo_service.get_merge_to_deploy_broker_bookmark(repo_id) - ) - if not merge_to_deploy_broker_bookmark: - merge_to_deploy_broker_bookmark = BookmarkMergeToDeployBroker( - repo_id=repo_id, bookmark=min_merged_time.isoformat() - ) - - self.code_repo_service.update_merge_to_deploy_broker_bookmark( - merge_to_deploy_broker_bookmark - ) - - -def get_merge_to_deploy_broker_utils_service(): - return MergeToDeployBrokerUtils( - CodeRepoService(), redis_lock_service=get_redis_lock_service() - ) diff --git a/apiserver/dora/service/pr_analytics.py b/apiserver/dora/service/pr_analytics.py deleted file mode 100644 index f540e3fa9..000000000 --- a/apiserver/dora/service/pr_analytics.py +++ /dev/null @@ -1,22 +0,0 @@ -from dora.store.models.code import OrgRepo, PullRequest -from dora.store.repos.code import CodeRepoService - -from typing import List - - -class PullRequestAnalyticsService: - def __init__(self, code_repo_service: CodeRepoService): - self.code_repo_service: CodeRepoService = code_repo_service - - def get_prs_by_ids(self, pr_ids: List[str]) -> List[PullRequest]: - return self.code_repo_service.get_prs_by_ids(pr_ids) - - def get_team_repos(self, team_id: str) -> List[OrgRepo]: - return self.code_repo_service.get_team_repos(team_id) - - def get_repo_by_id(self, team_id: str) -> List[OrgRepo]: - return self.code_repo_service.get_repo_by_id(team_id) - - -def get_pr_analytics_service(): - return PullRequestAnalyticsService(CodeRepoService()) diff --git a/apiserver/dora/service/query_validator.py b/apiserver/dora/service/query_validator.py deleted file mode 100644 index 1f6834bc6..000000000 --- a/apiserver/dora/service/query_validator.py +++ /dev/null @@ -1,77 +0,0 @@ -from datetime import timedelta -from typing import List - -from werkzeug.exceptions import NotFound, BadRequest - -from dora.store.models.core import Organization, Team, Users -from dora.store.repos.core import CoreRepoService -from dora.utils.time import Interval - -DEFAULT_ORG_NAME = "default" - - -class QueryValidator: - def __init__(self, repo_service: CoreRepoService): - self.repo_service = repo_service - - def get_default_org(self) -> Organization: - org: Organization = self.repo_service.get_org_by_name(DEFAULT_ORG_NAME) - if org is None: - raise NotFound("Default org not found") - return org - - def org_validator(self, org_id: str) -> Organization: - org: Organization = self.repo_service.get_org(org_id) - if org is None: - raise NotFound(f"Org {org_id} not found") - return org - - def team_validator(self, team_id: str) -> Team: - team: Team = self.repo_service.get_team(team_id) - if team is None: - raise NotFound(f"Team {team_id} not found") - return team - - def teams_validator(self, team_ids: List[str]) -> List[Team]: - teams: List[Team] = self.repo_service.get_teams(team_ids) - if len(teams) != len(team_ids): - query_team_ids = set(team_ids) - found_team_ids = set(map(lambda x: str(x.id), teams)) - missing_team_ids = query_team_ids - found_team_ids - raise NotFound(f"Team(s) not found: {missing_team_ids}") - return teams - - def interval_validator( - self, from_time, to_time, interval_limit_in_days: int = 105 - ) -> Interval: - if None in (from_time.tzinfo, to_time.tzinfo): - raise BadRequest("Timestamp passed without tz info") - interval = Interval(from_time, to_time) - if interval_limit_in_days is not None and interval.duration > timedelta( - days=interval_limit_in_days - ): - raise BadRequest( - f"Only {interval_limit_in_days} days duration is supported" - ) - return interval - - def user_validator(self, user_id: str) -> Users: - user = self.repo_service.get_user(user_id) - if user is None: - raise NotFound(f"User {user_id} not found") - return user - - def users_validator(self, user_ids: List[str]) -> List[Users]: - users: List[Users] = self.repo_service.get_users(user_ids) - - if len(users) != len(user_ids): - query_user_ids = set(user_ids) - found_user_ids = set(map(lambda x: str(x.id), users)) - missing_user_ids = query_user_ids - found_user_ids - raise NotFound(f"User(s) not found: {missing_user_ids}") - - return users - - -def get_query_validator(): - return QueryValidator(CoreRepoService()) diff --git a/apiserver/dora/service/settings/__init__.py b/apiserver/dora/service/settings/__init__.py deleted file mode 100644 index aca812775..000000000 --- a/apiserver/dora/service/settings/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .configuration_settings import SettingsService, get_settings_service -from .setting_type_validator import settings_type_validator diff --git a/apiserver/dora/service/settings/configuration_settings.py b/apiserver/dora/service/settings/configuration_settings.py deleted file mode 100644 index 4dc08c6da..000000000 --- a/apiserver/dora/service/settings/configuration_settings.py +++ /dev/null @@ -1,376 +0,0 @@ -from typing import Any, Dict, Optional, List - -from dora.service.settings.default_settings_data import get_default_setting_data -from dora.service.settings.models import ( - ConfigurationSettings, - ExcludedPRsSetting, - IncidentSettings, - IncidentSourcesSetting, - IncidentTypesSetting, -) -from dora.store.models.core.users import Users -from dora.store.models.incidents import IncidentSource, IncidentType -from dora.store.models.settings import SettingType, Settings, EntityType -from dora.store.repos.settings import SettingsRepoService -from dora.utils.time import time_now - - -class SettingsService: - def __init__(self, _settings_repo): - self._settings_repo: SettingsRepoService = _settings_repo - - def _adapt_specific_incident_setting_from_setting_data(self, data: Dict[str, any]): - """ - Adapts the json data in Settings.data to IncidentSettings - """ - - return IncidentSettings(title_filters=data.get("title_filters", [])) - - def _adapt_excluded_prs_setting_from_setting_data(self, data: Dict[str, any]): - """ - Adapts the json data in Setting.data for SettingType EXCLUDED_PRS_SETTING to ExcludedPRsSetting - """ - return ExcludedPRsSetting(excluded_pr_ids=data.get("excluded_pr_ids", [])) - - def _adapt_incident_source_setting_from_setting_data( - self, data: Dict[str, any] - ) -> IncidentSourcesSetting: - """ - Adapts the json data in Settings.data to IncidentSourcesSetting - """ - return IncidentSourcesSetting( - incident_sources=[ - IncidentSource(source) for source in data.get("incident_sources") or [] - ] - ) - - def _adapt_incident_types_setting_from_setting_data( - self, data: Dict[str, any] - ) -> IncidentTypesSetting: - """ - Adapts the json data in Settings.data to IncidentTypesSetting - """ - - return IncidentTypesSetting( - incident_types=[ - IncidentType(incident_type) - for incident_type in data.get("incident_types") or [] - ] - ) - - def _handle_config_setting_from_db_setting( - self, setting_type: SettingType, setting_data - ): - # Add if statements and adapters for new setting types - - if setting_type == SettingType.INCIDENT_SETTING: - return self._adapt_specific_incident_setting_from_setting_data(setting_data) - - if setting_type == SettingType.EXCLUDED_PRS_SETTING: - return self._adapt_excluded_prs_setting_from_setting_data(setting_data) - - if setting_type == SettingType.INCIDENT_TYPES_SETTING: - return self._adapt_incident_types_setting_from_setting_data(setting_data) - - if setting_type == SettingType.INCIDENT_SOURCES_SETTING: - return self._adapt_incident_source_setting_from_setting_data(setting_data) - - raise Exception(f"Invalid Setting Type: {setting_type}") - - def _adapt_config_setting_from_db_setting(self, setting: Settings): - specific_setting = self._handle_config_setting_from_db_setting( - setting.setting_type, setting.data - ) - - return ConfigurationSettings( - entity_id=setting.entity_id, - entity_type=setting.entity_type, - updated_by=setting.updated_by, - created_at=setting.created_at, - updated_at=setting.updated_at, - specific_settings=specific_setting, - ) - - def get_settings( - self, setting_type: SettingType, entity_type: EntityType, entity_id: str - ) -> Optional[ConfigurationSettings]: - - setting = self._settings_repo.get_setting( - entity_id=entity_id, - entity_type=entity_type, - setting_type=setting_type, - ) - if not setting: - return None - - return self._adapt_config_setting_from_db_setting(setting) - - def get_or_set_settings_for_multiple_entity_ids( - self, - setting_type: SettingType, - entity_type: EntityType, - entity_ids: List[str], - setter: Users = None, - ) -> List[ConfigurationSettings]: - - settings = self._settings_repo.get_settings_for_multiple_entity_ids( - entity_ids, entity_type, setting_type - ) - - current_entity_ids = set([str(setting.entity_id) for setting in settings]) - missing_entity_ids = set(entity_ids).difference(current_entity_ids) - if missing_entity_ids: - data = get_default_setting_data(setting_type) - settings_to_create = [ - Settings( - entity_id=entity_id, - entity_type=entity_type, - setting_type=setting_type, - updated_by=setter.id if setter else None, - data=data, - created_at=time_now(), - updated_at=time_now(), - is_deleted=False, - ) - for entity_id in missing_entity_ids - ] - new_settings = self._settings_repo.create_settings(settings_to_create) - settings.extend(new_settings) - - return list(map(self._adapt_config_setting_from_db_setting, settings)) - - def _adapt_specific_incident_setting_from_json( - self, data: Dict[str, any] - ) -> IncidentSettings: - """ - Adapts the json data from API to IncidentSettings - """ - - return IncidentSettings(title_filters=data.get("title_includes", [])) - - def _adapt_excluded_prs_setting_from_json(self, data: Dict[str, any]): - """ - Adapts the json data from API for SettingType EXCLUDED_PRS_SETTING to ExcludedPrsSetting - """ - return ExcludedPRsSetting(excluded_pr_ids=data.get("excluded_pr_ids", [])) - - def _adapt_incident_source_setting_from_json( - self, data: Dict[str, any] - ) -> IncidentSourcesSetting: - """ - Adapts the json data from API to IncidentSourcesSetting - """ - - return IncidentSourcesSetting( - incident_sources=[ - IncidentSource(source) for source in data.get("incident_sources") or [] - ] - ) - - def _adapt_incident_types_setting_from_json( - self, data: Dict[str, any] - ) -> IncidentTypesSetting: - """ - Adapts the json data from API to IncidentTypesSetting - """ - - return IncidentTypesSetting( - incident_types=[ - IncidentType(incident_type) - for incident_type in data.get("incident_types") or [] - ] - ) - - def _handle_config_setting_from_json_data( - self, setting_type: SettingType, setting_data - ): - # Add if statements and adapters for new setting types - - if setting_type == SettingType.INCIDENT_SETTING: - return self._adapt_specific_incident_setting_from_json(setting_data) - - if setting_type == SettingType.EXCLUDED_PRS_SETTING: - return self._adapt_excluded_prs_setting_from_json(setting_data) - - if setting_type == SettingType.INCIDENT_SOURCES_SETTING: - return self._adapt_incident_source_setting_from_json(setting_data) - - if setting_type == SettingType.INCIDENT_TYPES_SETTING: - return self._adapt_incident_types_setting_from_json(setting_data) - - raise Exception(f"Invalid Setting Type: {setting_type}") - - def _adapt_incident_setting_json_data( - self, - specific_setting: IncidentSettings, - ): - return {"title_filters": specific_setting.title_filters} - - def _adapt_excluded_prs_setting_json_data( - self, specific_setting: ExcludedPRsSetting - ): - return {"excluded_pr_ids": specific_setting.excluded_pr_ids} - - def _adapt_incident_source_setting_json_data( - self, specific_setting: IncidentSourcesSetting - ) -> Dict: - return { - "incident_sources": [ - source.value for source in specific_setting.incident_sources - ] - } - - def _adapt_incident_types_setting_json_data( - self, specific_setting: IncidentTypesSetting - ) -> Dict: - return { - "incident_types": [ - incident_type.value for incident_type in specific_setting.incident_types - ] - } - - def _handle_config_setting_to_db_setting( - self, setting_type: SettingType, specific_setting - ): - # Add if statements and adapters to get data for new setting types - - if setting_type == SettingType.INCIDENT_SETTING and isinstance( - specific_setting, IncidentSettings - ): - return self._adapt_incident_setting_json_data(specific_setting) - if setting_type == SettingType.EXCLUDED_PRS_SETTING and isinstance( - specific_setting, ExcludedPRsSetting - ): - return self._adapt_excluded_prs_setting_json_data(specific_setting) - - if setting_type == SettingType.INCIDENT_TYPES_SETTING and isinstance( - specific_setting, IncidentTypesSetting - ): - return self._adapt_incident_types_setting_json_data(specific_setting) - - if setting_type == SettingType.INCIDENT_SOURCES_SETTING and isinstance( - specific_setting, IncidentSourcesSetting - ): - return self._adapt_incident_source_setting_json_data(specific_setting) - - raise Exception(f"Invalid Setting Type: {setting_type}") - - def _adapt_specific_setting_data_from_json( - self, setting_type: SettingType, setting_data: dict - ): - """ - This function is getting json data (setting_data) and adapting it to the data class as per the setting type. - This then again converts the class data into a dictionary and returns it. - - The process has been done in order to just maintain the data sanctity and to avoid any un-formatted data being stored in the DB. - """ - - specific_setting = self._handle_config_setting_from_json_data( - setting_type, setting_data - ) - - return self._handle_config_setting_to_db_setting(setting_type, specific_setting) - - def save_settings( - self, - setting_type: SettingType, - entity_type: EntityType, - entity_id: str, - setter: Users = None, - setting_data: Dict = None, - ) -> ConfigurationSettings: - - if setting_data: - data = self._adapt_specific_setting_data_from_json( - setting_type, setting_data - ) - else: - data = get_default_setting_data(setting_type) - - setting = Settings( - entity_id=entity_id, - entity_type=entity_type, - setting_type=setting_type, - updated_by=setter.id if setter else None, - data=data, - created_at=time_now(), - updated_at=time_now(), - is_deleted=False, - ) - - saved_setting = self._settings_repo.save_setting(setting) - - return self._adapt_config_setting_from_db_setting(saved_setting) - - def delete_settings( - self, - setting_type: SettingType, - entity_type: EntityType, - deleted_by: Users, - entity_id: str, - ) -> ConfigurationSettings: - - return self._adapt_config_setting_from_db_setting( - self._settings_repo.delete_setting( - setting_type=setting_type, - entity_id=entity_id, - entity_type=entity_type, - deleted_by=deleted_by, - ) - ) - - def get_settings_map( - self, - entity_id: str, - setting_types: List[SettingType], - entity_type: EntityType, - ignore_default_setting_type: List[SettingType] = None, - ) -> Dict[SettingType, any]: - - if not ignore_default_setting_type: - ignore_default_setting_type = [] - - settings: List[Settings] = self._settings_repo.get_settings( - entity_id=entity_id, setting_types=setting_types, entity_type=entity_type - ) - setting_type_to_setting_map: Dict[ - SettingType, Any - ] = self._get_setting_type_to_setting_map( - setting_types, settings, ignore_default_setting_type - ) - - return setting_type_to_setting_map - - def _get_setting_type_to_setting_map( - self, - setting_types: List[SettingType], - settings: List[Settings], - ignore_default_setting_type: List[SettingType] = None, - ) -> Dict[SettingType, Any]: - - if not ignore_default_setting_type: - ignore_default_setting_type = [] - - setting_type_to_setting_map: Dict[SettingType, Any] = {} - for setting in settings: - setting_type_to_setting_map[ - setting.setting_type - ] = self._adapt_config_setting_from_db_setting(setting).specific_settings - - for setting_type in setting_types: - if (setting_type not in setting_type_to_setting_map) and ( - setting_type not in ignore_default_setting_type - ): - setting_type_to_setting_map[setting_type] = self.get_default_setting( - setting_type - ) - return setting_type_to_setting_map - - def get_default_setting(self, setting_type: SettingType): - return self._handle_config_setting_from_db_setting( - setting_type, get_default_setting_data(setting_type) - ) - - -def get_settings_service(): - return SettingsService(SettingsRepoService()) diff --git a/apiserver/dora/service/settings/default_settings_data.py b/apiserver/dora/service/settings/default_settings_data.py deleted file mode 100644 index c9ec1c0e9..000000000 --- a/apiserver/dora/service/settings/default_settings_data.py +++ /dev/null @@ -1,29 +0,0 @@ -from dora.store.models.incidents import IncidentSource, IncidentType -from dora.store.models.settings import SettingType - - -MIN_CYCLE_TIME_THRESHOLD = 3600 - - -def get_default_setting_data(setting_type: SettingType): - if setting_type == SettingType.INCIDENT_SETTING: - return {"title_filters": []} - - if setting_type == SettingType.EXCLUDED_PRS_SETTING: - return {"excluded_pr_ids": []} - - if setting_type == SettingType.INCIDENT_SOURCES_SETTING: - incident_sources = list(IncidentSource) - return { - "incident_sources": [ - incident_source.value for incident_source in incident_sources - ] - } - - if setting_type == SettingType.INCIDENT_TYPES_SETTING: - incident_types = list(IncidentType) - return { - "incident_types": [incident_type.value for incident_type in incident_types] - } - - raise Exception(f"Invalid Setting Type: {setting_type}") diff --git a/apiserver/dora/service/settings/models.py b/apiserver/dora/service/settings/models.py deleted file mode 100644 index 0bbfff43c..000000000 --- a/apiserver/dora/service/settings/models.py +++ /dev/null @@ -1,41 +0,0 @@ -from dataclasses import dataclass -from datetime import datetime -from typing import List - -from dora.store.models import EntityType -from dora.store.models.incidents.enums import IncidentSource, IncidentType - - -@dataclass -class BaseSetting: - pass - - -@dataclass -class ConfigurationSettings: - entity_id: str - entity_type: EntityType - specific_settings: BaseSetting - updated_by: str - created_at: datetime - updated_at: datetime - - -@dataclass -class IncidentSettings(BaseSetting): - title_filters: List[str] - - -@dataclass -class ExcludedPRsSetting(BaseSetting): - excluded_pr_ids: List[str] - - -@dataclass -class IncidentTypesSetting(BaseSetting): - incident_types: List[IncidentType] - - -@dataclass -class IncidentSourcesSetting(BaseSetting): - incident_sources: List[IncidentSource] diff --git a/apiserver/dora/service/settings/setting_type_validator.py b/apiserver/dora/service/settings/setting_type_validator.py deleted file mode 100644 index 1e8415de2..000000000 --- a/apiserver/dora/service/settings/setting_type_validator.py +++ /dev/null @@ -1,19 +0,0 @@ -from werkzeug.exceptions import BadRequest - -from dora.store.models.settings import SettingType - - -def settings_type_validator(setting_type: str): - if setting_type == SettingType.INCIDENT_SETTING.value: - return SettingType.INCIDENT_SETTING - - if setting_type == SettingType.EXCLUDED_PRS_SETTING.value: - return SettingType.EXCLUDED_PRS_SETTING - - if setting_type == SettingType.INCIDENT_TYPES_SETTING.value: - return SettingType.INCIDENT_TYPES_SETTING - - if setting_type == SettingType.INCIDENT_SOURCES_SETTING.value: - return SettingType.INCIDENT_SOURCES_SETTING - - raise BadRequest(f"Invalid Setting Type: {setting_type}") diff --git a/apiserver/dora/service/sync_data.py b/apiserver/dora/service/sync_data.py deleted file mode 100644 index d5accbfa5..000000000 --- a/apiserver/dora/service/sync_data.py +++ /dev/null @@ -1,34 +0,0 @@ -from dora.service.code import sync_code_repos -from dora.service.incidents import sync_org_incidents -from dora.service.merge_to_deploy_broker import process_merge_to_deploy_cache -from dora.service.query_validator import get_query_validator -from dora.service.workflows import sync_org_workflows -from dora.utils.log import LOG - -sync_sequence = [ - sync_code_repos, - sync_org_workflows, - process_merge_to_deploy_cache, - sync_org_incidents, -] - - -def trigger_data_sync(): - default_org = get_query_validator().get_default_org() - org_id = str(default_org.id) - LOG.info(f"Starting data sync for org {org_id}") - - for sync_func in sync_sequence: - try: - sync_func(org_id) - LOG.info(f"Data sync for {sync_func.__name__} completed successfully") - except Exception as e: - LOG.error( - f"Error syncing {sync_func.__name__} data for org {org_id}: {str(e)}" - ) - continue - LOG.info(f"Data sync for org {org_id} completed successfully") - - -if __name__ == "__main__": - trigger_data_sync() diff --git a/apiserver/dora/service/workflows/__init__.py b/apiserver/dora/service/workflows/__init__.py deleted file mode 100644 index c4ede6824..000000000 --- a/apiserver/dora/service/workflows/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .sync import sync_org_workflows diff --git a/apiserver/dora/service/workflows/integration.py b/apiserver/dora/service/workflows/integration.py deleted file mode 100644 index 6f4df4f9d..000000000 --- a/apiserver/dora/service/workflows/integration.py +++ /dev/null @@ -1,28 +0,0 @@ -from typing import List - -from dora.store.models import Integration -from dora.store.models.code import RepoWorkflowProviders -from dora.store.repos.core import CoreRepoService - -WORKFLOW_INTEGRATION_BUCKET = [ - RepoWorkflowProviders.GITHUB_ACTIONS.value, -] - - -class WorkflowsIntegrationsService: - def __init__(self, core_repo_service: CoreRepoService): - self.core_repo_service = core_repo_service - - def get_org_providers(self, org_id: str) -> List[str]: - integrations: List[ - Integration - ] = self.core_repo_service.get_org_integrations_for_names( - org_id, WORKFLOW_INTEGRATION_BUCKET - ) - if not integrations: - return [] - return [integration.name for integration in integrations] - - -def get_workflows_integrations_service() -> WorkflowsIntegrationsService: - return WorkflowsIntegrationsService(CoreRepoService()) diff --git a/apiserver/dora/service/workflows/sync/__init__.py b/apiserver/dora/service/workflows/sync/__init__.py deleted file mode 100644 index 3a36308d5..000000000 --- a/apiserver/dora/service/workflows/sync/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .etl_handler import sync_org_workflows diff --git a/apiserver/dora/service/workflows/sync/etl_github_actions_handler.py b/apiserver/dora/service/workflows/sync/etl_github_actions_handler.py deleted file mode 100644 index 2ce3bd431..000000000 --- a/apiserver/dora/service/workflows/sync/etl_github_actions_handler.py +++ /dev/null @@ -1,189 +0,0 @@ -from datetime import datetime -from typing import Dict, Optional, List, Tuple -from uuid import uuid4 - -import pytz - -from dora.exapi.github import GithubApiService -from dora.service.workflows.sync.etl_provider_handler import WorkflowProviderETLHandler -from dora.store.models import UserIdentityProvider -from dora.store.models.code import ( - RepoWorkflowProviders, - RepoWorkflowRunsStatus, - RepoWorkflowRuns, - OrgRepo, - RepoWorkflowRunsBookmark, - RepoWorkflow, -) -from dora.store.repos.core import CoreRepoService -from dora.store.repos.workflows import WorkflowRepoService -from dora.utils.log import LOG -from dora.utils.time import ISO_8601_DATE_FORMAT, time_now - -DEFAULT_WORKFLOW_SYNC_DAYS = 31 -WORKFLOW_PROCESSING_CHUNK_SIZE = 100 - - -class GithubActionsETLHandler(WorkflowProviderETLHandler): - def __init__( - self, - org_id: str, - github_api_service: GithubApiService, - workflow_repo_service: WorkflowRepoService, - ): - self.org_id = org_id - self._api: GithubApiService = github_api_service - self._workflow_repo_service = workflow_repo_service - self._provider = RepoWorkflowProviders.GITHUB_ACTIONS.value - - def check_pat_validity(self) -> bool: - """ - This method checks if the PAT is valid. - :returns: PAT details - :raises: Exception if PAT is invalid - """ - is_valid = self._api.check_pat() - if not is_valid: - raise Exception("Github Personal Access Token is invalid") - return is_valid - - def get_workflow_runs( - self, - org_repo: OrgRepo, - repo_workflow: RepoWorkflow, - bookmark: RepoWorkflowRunsBookmark, - ) -> Tuple[List[RepoWorkflowRuns], RepoWorkflowRunsBookmark]: - """ - This method returns all workflow runs of a repo's workflow. After the bookmark date. - :param org_repo: OrgRepo object to get workflow runs for - :param repo_workflow: RepoWorkflow object to get workflow runs for - :param bookmark: Bookmark object to get all workflow runs after this date - :return: Workflow runs, Bookmark object - """ - bookmark_time_stamp = datetime.fromisoformat(bookmark.bookmark) - try: - github_workflow_runs = self._api.get_workflow_runs( - org_repo.org_name, - org_repo.name, - repo_workflow.provider_workflow_id, - bookmark_time_stamp, - ) - except Exception as e: - raise Exception( - f"[GitHub Sync Repo Workflow Worker] Error fetching workflow {str(repo_workflow.id)} " - f"for repo {str(org_repo.repo_id)}: {str(e)}" - ) - - if not github_workflow_runs: - LOG.info( - f"[GitHub Sync Repo Workflow Worker] No Workflow Runs found for " - f"Workflow: {str(repo_workflow.provider_workflow_id)}. Repo: {org_repo.org_name}/{org_repo.name}. " - f"Org: {self.org_id}" - ) - return [], bookmark - - bookmark.bookmark = self._get_new_bookmark_time_stamp( - github_workflow_runs - ).isoformat() - - repo_workflow_runs = [ - self._adapt_github_workflows_to_workflow_runs( - str(repo_workflow.id), workflow_run - ) - for workflow_run in github_workflow_runs - ] - - return repo_workflow_runs, bookmark - - def _get_new_bookmark_time_stamp( - self, github_workflow_runs: List[Dict] - ) -> datetime: - """ - This method returns the new bookmark timestamp for the workflow runs. - It returns the minimum timestamp of the pending jobs if there are any pending jobs. - This is done because there might be a workflow run that is still pending, and we - want to fetch it in the next sync. - """ - pending_job_timestamps = [ - self._get_datetime_from_gh_datetime(workflow_run["created_at"]) - for workflow_run in github_workflow_runs - if workflow_run["status"] != "completed" - ] - return min(pending_job_timestamps) if pending_job_timestamps else time_now() - - def _adapt_github_workflows_to_workflow_runs( - self, repo_workflow_id: str, github_workflow_run: Dict - ) -> RepoWorkflowRuns: - repo_workflow_run_in_db = self._workflow_repo_service.get_repo_workflow_run_by_provider_workflow_run_id( - repo_workflow_id, str(github_workflow_run["id"]) - ) - if repo_workflow_run_in_db: - workflow_run_id = repo_workflow_run_in_db.id - else: - workflow_run_id = uuid4() - return RepoWorkflowRuns( - id=workflow_run_id, - repo_workflow_id=repo_workflow_id, - provider_workflow_run_id=str(github_workflow_run["id"]), - event_actor=github_workflow_run["actor"]["login"], - head_branch=github_workflow_run["head_branch"], - status=self._get_repo_workflow_status(github_workflow_run), - created_at=time_now(), - updated_at=time_now(), - conducted_at=self._get_datetime_from_gh_datetime( - github_workflow_run["run_started_at"] - ), - duration=self._get_repo_workflow_run_duration(github_workflow_run), - meta=github_workflow_run, - html_url=github_workflow_run["html_url"], - ) - - @staticmethod - def _get_repo_workflow_status(github_workflow: Dict) -> RepoWorkflowRunsStatus: - if github_workflow["status"] != "completed": - return RepoWorkflowRunsStatus.PENDING - if github_workflow["conclusion"] == "success": - return RepoWorkflowRunsStatus.SUCCESS - return RepoWorkflowRunsStatus.FAILURE - - def _get_repo_workflow_run_duration( - self, github_workflow_run: Dict - ) -> Optional[int]: - - if not ( - github_workflow_run.get("updated_at") - and github_workflow_run.get("run_started_at") - ): - return None - - workflow_run_updated_at = self._get_datetime_from_gh_datetime( - github_workflow_run.get("updated_at") - ) - workflow_run_conducted_at = self._get_datetime_from_gh_datetime( - github_workflow_run.get("run_started_at") - ) - return int( - (workflow_run_updated_at - workflow_run_conducted_at).total_seconds() - ) - - @staticmethod - def _get_datetime_from_gh_datetime(datetime_str: str) -> datetime: - dt_without_timezone = datetime.strptime(datetime_str, ISO_8601_DATE_FORMAT) - return dt_without_timezone.replace(tzinfo=pytz.UTC) - - -def get_github_actions_etl_handler(org_id): - def _get_access_token(): - core_repo_service = CoreRepoService() - access_token = core_repo_service.get_access_token( - org_id, UserIdentityProvider.GITHUB - ) - if not access_token: - raise Exception( - f"Access token not found for org {org_id} and provider {UserIdentityProvider.GITHUB.value}" - ) - return access_token - - return GithubActionsETLHandler( - org_id, GithubApiService(_get_access_token()), WorkflowRepoService() - ) diff --git a/apiserver/dora/service/workflows/sync/etl_handler.py b/apiserver/dora/service/workflows/sync/etl_handler.py deleted file mode 100644 index a813778bd..000000000 --- a/apiserver/dora/service/workflows/sync/etl_handler.py +++ /dev/null @@ -1,134 +0,0 @@ -from datetime import timedelta -from typing import List, Tuple -from uuid import uuid4 - -from dora.service.code import get_code_integration_service -from dora.service.workflows.integration import get_workflows_integrations_service -from dora.service.workflows.sync.etl_provider_handler import WorkflowProviderETLHandler -from dora.service.workflows.sync.etl_workflows_factory import WorkflowETLFactory -from dora.store.models.code import ( - OrgRepo, - RepoWorkflow, - RepoWorkflowRunsBookmark, - RepoWorkflowRuns, - RepoWorkflowProviders, -) -from dora.store.repos.code import CodeRepoService -from dora.store.repos.workflows import WorkflowRepoService -from dora.utils.log import LOG -from dora.utils.time import time_now - - -class WorkflowETLHandler: - def __init__( - self, - code_repo_service: CodeRepoService, - workflow_repo_service: WorkflowRepoService, - etl_factory: WorkflowETLFactory, - ): - self.code_repo_service = code_repo_service - self.workflow_repo_service = workflow_repo_service - self.etl_factory = etl_factory - - def sync_org_workflows(self, org_id: str): - active_repo_workflows: List[ - Tuple[OrgRepo, RepoWorkflow] - ] = self._get_active_repo_workflows(org_id) - - for org_repo, repo_workflow in active_repo_workflows: - try: - self._sync_repo_workflow(org_repo, repo_workflow) - except Exception as e: - LOG.error( - f"Error syncing workflow for repo {repo_workflow.org_repo_id}: {str(e)}" - ) - continue - - def _get_active_repo_workflows( - self, org_id: str - ) -> List[Tuple[OrgRepo, RepoWorkflow]]: - code_providers: List[str] = get_code_integration_service().get_org_providers( - org_id - ) - workflow_providers: List[ - str - ] = get_workflows_integrations_service().get_org_providers(org_id) - if not code_providers or not workflow_providers: - LOG.info(f"No workflow integrations found for org {org_id}") - return [] - - org_repos: List[OrgRepo] = self.code_repo_service.get_active_org_repos(org_id) - repo_ids = [str(repo.id) for repo in org_repos] - repo_id_org_repo_map = {str(repo.id): repo for repo in org_repos} - active_repo_workflows: List[ - RepoWorkflow - ] = self.workflow_repo_service.get_active_repo_workflows_by_repo_ids_and_providers( - repo_ids, - [RepoWorkflowProviders(provider) for provider in workflow_providers], - ) - org_repo_workflows: List[Tuple[OrgRepo, RepoWorkflow]] = [] - for repo_workflow in active_repo_workflows: - org_repo_workflows.append( - (repo_id_org_repo_map[str(repo_workflow.org_repo_id)], repo_workflow) - ) - return org_repo_workflows - - def _sync_repo_workflow(self, org_repo: OrgRepo, repo_workflow: RepoWorkflow): - workflow_provider: RepoWorkflowProviders = repo_workflow.provider - etl_service: WorkflowProviderETLHandler = self.etl_factory( - workflow_provider.name - ) - if not etl_service.check_pat_validity(): - LOG.error("Invalid PAT for code provider") - return - try: - bookmark: RepoWorkflowRunsBookmark = self.__get_repo_workflow_bookmark( - repo_workflow - ) - repo_workflow_runs: List[RepoWorkflowRuns] - repo_workflow_runs, bookmark = etl_service.get_workflow_runs( - org_repo, repo_workflow, bookmark - ) - self.workflow_repo_service.save_repo_workflow_runs(repo_workflow_runs) - self.workflow_repo_service.update_repo_workflow_runs_bookmark(bookmark) - except Exception as e: - LOG.error( - f"Error syncing workflow for repo {repo_workflow.org_repo_id}: {str(e)}" - ) - return - - def __get_repo_workflow_bookmark( - self, repo_workflow: RepoWorkflow, default_sync_days: int = 31 - ) -> RepoWorkflowRunsBookmark: - repo_workflow_bookmark = ( - self.workflow_repo_service.get_repo_workflow_runs_bookmark(repo_workflow.id) - ) - if not repo_workflow_bookmark: - bookmark_string = ( - time_now() - timedelta(days=default_sync_days) - ).isoformat() - - repo_workflow_bookmark = RepoWorkflowRunsBookmark( - id=uuid4(), - repo_workflow_id=repo_workflow.id, - bookmark=bookmark_string, - created_at=time_now(), - updated_at=time_now(), - ) - return repo_workflow_bookmark - - -def sync_org_workflows(org_id: str): - workflow_providers: List[ - str - ] = get_workflows_integrations_service().get_org_providers(org_id) - if not workflow_providers: - LOG.info(f"No workflow integrations found for org {org_id}") - return - code_repo_service = CodeRepoService() - workflow_repo_service = WorkflowRepoService() - etl_factory = WorkflowETLFactory(org_id) - workflow_etl_handler = WorkflowETLHandler( - code_repo_service, workflow_repo_service, etl_factory - ) - workflow_etl_handler.sync_org_workflows(org_id) diff --git a/apiserver/dora/service/workflows/sync/etl_provider_handler.py b/apiserver/dora/service/workflows/sync/etl_provider_handler.py deleted file mode 100644 index 72b56aece..000000000 --- a/apiserver/dora/service/workflows/sync/etl_provider_handler.py +++ /dev/null @@ -1,36 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List, Tuple - -from dora.store.models.code import ( - OrgRepo, - RepoWorkflow, - RepoWorkflowRunsBookmark, - RepoWorkflowRuns, -) - - -class WorkflowProviderETLHandler(ABC): - @abstractmethod - def check_pat_validity(self) -> bool: - """ - This method checks if the PAT is valid. - :return: PAT details - :raises: Exception if PAT is invalid - """ - pass - - @abstractmethod - def get_workflow_runs( - self, - org_repo: OrgRepo, - repo_workflow: RepoWorkflow, - bookmark: RepoWorkflowRunsBookmark, - ) -> Tuple[List[RepoWorkflowRuns], RepoWorkflowRunsBookmark]: - """ - This method returns all workflow runs of a repo's workflow. After the bookmark date. - :param org_repo: OrgRepo object to get workflow runs for - :param repo_workflow: RepoWorkflow object to get workflow runs for - :param bookmark: Bookmark object to get all workflow runs after this date - :return: List of RepoWorkflowRuns objects, RepoWorkflowRunsBookmark object - """ - pass diff --git a/apiserver/dora/service/workflows/sync/etl_workflows_factory.py b/apiserver/dora/service/workflows/sync/etl_workflows_factory.py deleted file mode 100644 index 8aadb02d8..000000000 --- a/apiserver/dora/service/workflows/sync/etl_workflows_factory.py +++ /dev/null @@ -1,15 +0,0 @@ -from dora.service.workflows.sync.etl_github_actions_handler import ( - get_github_actions_etl_handler, -) -from dora.service.workflows.sync.etl_provider_handler import WorkflowProviderETLHandler -from dora.store.models.code import RepoWorkflowProviders - - -class WorkflowETLFactory: - def __init__(self, org_id: str): - self.org_id = org_id - - def __call__(self, provider: str) -> WorkflowProviderETLHandler: - if provider == RepoWorkflowProviders.GITHUB_ACTIONS.name: - return get_github_actions_etl_handler(self.org_id) - raise NotImplementedError(f"Unknown provider - {provider}") diff --git a/apiserver/dora/service/workflows/workflow_filter.py b/apiserver/dora/service/workflows/workflow_filter.py deleted file mode 100644 index a693687ca..000000000 --- a/apiserver/dora/service/workflows/workflow_filter.py +++ /dev/null @@ -1,52 +0,0 @@ -import json - -from typing import List, Dict - -from sqlalchemy import or_ - -from dora.store.models.code.workflows.filter import WorkflowFilter - - -class ParseWorkflowFilterProcessor: - def apply(self, workflow_filter: Dict = None) -> WorkflowFilter: - head_branches: List[str] = self._parse_head_branches(workflow_filter) - repo_filters: Dict[str, Dict] = self._parse_repo_filters(workflow_filter) - - return WorkflowFilter( - head_branches=head_branches, - repo_filters=repo_filters, - ) - - def _parse_head_branches(self, workflow_filter: Dict) -> List[str]: - return workflow_filter.get("head_branches") - - def _parse_repo_filters(self, workflow_filter: Dict) -> Dict[str, Dict]: - repo_filters: Dict[str, Dict] = workflow_filter.get("repo_filters") - if repo_filters: - for repo_id, repo_filter in repo_filters.items(): - repo_head_branches: List[str] = self._parse_repo_head_branches( - repo_filter - ) - repo_filters[repo_id]["head_branches"] = repo_head_branches - return repo_filters - - def _parse_repo_head_branches(self, repo_filter: Dict[str, any]) -> List[str]: - repo_head_branches: List[str] = repo_filter.get("head_branches") - if not repo_head_branches: - return [] - return repo_head_branches - - -class WorkflowFilterProcessor: - def __init__(self, parse_workflow_filter_processor: ParseWorkflowFilterProcessor): - self.parse_workflow_filter_processor = parse_workflow_filter_processor - - def create_workflow_filter_from_json_string( - self, filter_data: str - ) -> WorkflowFilter: - filter_data = filter_data or "{}" - return self.parse_workflow_filter_processor.apply(json.loads(filter_data)) - - -def get_workflow_filter_processor() -> WorkflowFilterProcessor: - return WorkflowFilterProcessor(ParseWorkflowFilterProcessor()) diff --git a/apiserver/dora/store/__init__.py b/apiserver/dora/store/__init__.py deleted file mode 100644 index 0ec5170c5..000000000 --- a/apiserver/dora/store/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -from os import getenv - -from flask_sqlalchemy import SQLAlchemy - -from dora.utils.log import LOG - -db = SQLAlchemy() - - -def configure_db_with_app(app): - - DB_HOST = getenv("DB_HOST") - DB_PORT = getenv("DB_PORT") - DB_USER = getenv("DB_USER") - DB_PASS = getenv("DB_PASS") - DB_NAME = getenv("DB_NAME") - ENVIRONMENT = getenv("ENVIRONMENT", "local") - - connection_uri = f"postgresql://{DB_USER}:{DB_PASS}@{DB_HOST}:{DB_PORT}/{DB_NAME}?application_name=dora--{ENVIRONMENT}" - - app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False - app.config["SQLALCHEMY_DATABASE_URI"] = connection_uri - app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {"pool_size": 10, "max_overflow": 5} - db.init_app(app) - - -def rollback_on_exc(func): - def wrapper(self, *args, **kwargs): - try: - return func(self, *args, **kwargs) - except Exception as e: - self._db.session.rollback() - LOG.error(f"Error in {func.__name__} - {str(e)}") - raise - - return wrapper diff --git a/apiserver/dora/store/initialise_db.py b/apiserver/dora/store/initialise_db.py deleted file mode 100644 index 7439662b8..000000000 --- a/apiserver/dora/store/initialise_db.py +++ /dev/null @@ -1,27 +0,0 @@ -from dora.store import db -from dora.store.models import Organization -from dora.utils.string import uuid4_str -from dora.utils.time import time_now - - -def initialize_database(app): - with app.app_context(): - default_org = ( - db.session.query(Organization) - .filter(Organization.name == "default") - .one_or_none() - ) - if default_org: - return - default_org = Organization( - id=uuid4_str(), - name="default", - domain="default", - created_at=time_now(), - ) - db.session.add(default_org) - db.session.commit() - - -if __name__ == "__main__": - initialize_database() diff --git a/apiserver/dora/store/models/__init__.py b/apiserver/dora/store/models/__init__.py deleted file mode 100644 index dca8100bd..000000000 --- a/apiserver/dora/store/models/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .core import Organization, Team, Users -from .integrations import Integration, UserIdentity, UserIdentityProvider -from .settings import ( - EntityType, - Settings, - SettingType, -) diff --git a/apiserver/dora/store/models/code/__init__.py b/apiserver/dora/store/models/code/__init__.py deleted file mode 100644 index 3552af427..000000000 --- a/apiserver/dora/store/models/code/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -from .enums import ( - CodeProvider, - BookmarkType, - PullRequestState, - PullRequestEventState, - PullRequestEventType, - PullRequestRevertPRMappingActorType, -) -from .filter import PRFilter -from .pull_requests import ( - PullRequest, - PullRequestEvent, - PullRequestCommit, - PullRequestRevertPRMapping, -) -from .repository import ( - OrgRepo, - TeamRepos, - RepoSyncLogs, - Bookmark, - BookmarkMergeToDeployBroker, -) -from .workflows import ( - RepoWorkflow, - RepoWorkflowRuns, - RepoWorkflowRunsBookmark, - RepoWorkflowType, - RepoWorkflowProviders, - RepoWorkflowRunsStatus, - WorkflowFilter, -) diff --git a/apiserver/dora/store/models/code/enums.py b/apiserver/dora/store/models/code/enums.py deleted file mode 100644 index 5c9512aac..000000000 --- a/apiserver/dora/store/models/code/enums.py +++ /dev/null @@ -1,35 +0,0 @@ -from enum import Enum - - -class CodeProvider(Enum): - GITHUB = "github" - - -class BookmarkType(Enum): - PR = "PR" - - -class TeamReposDeploymentType(Enum): - WORKFLOW = "WORKFLOW" - PR_MERGE = "PR_MERGE" - - -class PullRequestState(Enum): - OPEN = "OPEN" - CLOSED = "CLOSED" - MERGED = "MERGED" - - -class PullRequestEventState(Enum): - CHANGES_REQUESTED = "CHANGES_REQUESTED" - APPROVED = "APPROVED" - COMMENTED = "COMMENTED" - - -class PullRequestEventType(Enum): - REVIEW = "REVIEW" - - -class PullRequestRevertPRMappingActorType(Enum): - SYSTEM = "SYSTEM" - USER = "USER" diff --git a/apiserver/dora/store/models/code/filter.py b/apiserver/dora/store/models/code/filter.py deleted file mode 100644 index 422f8fd3b..000000000 --- a/apiserver/dora/store/models/code/filter.py +++ /dev/null @@ -1,98 +0,0 @@ -from dataclasses import dataclass -from typing import List, Dict - -from sqlalchemy import and_, or_ - -from dora.store.models.code.pull_requests import PullRequest - - -@dataclass -class PRFilter: - authors: List[str] = None - base_branches: List[str] = None - repo_filters: Dict[str, Dict] = None - excluded_pr_ids: List[str] = None - max_cycle_time: int = None - - class RepoFilter: - def __init__(self, repo_id: str, repo_filters=None): - if repo_filters is None: - repo_filters = {} - self.repo_id = repo_id - self.base_branches = repo_filters.get("base_branches", []) - - @property - def filter_query(self): - def _repo_id_query(): - if not self.repo_id: - raise ValueError("repo_id is required") - return PullRequest.repo_id == self.repo_id - - def _base_branch_query(): - if not self.base_branches: - return None - return or_( - PullRequest.base_branch.op("~")(term) - for term in self.base_branches - if term is not None - ) - - conditions = { - "repo_id": _repo_id_query(), - "base_branches": _base_branch_query(), - } - queries = [ - conditions[x] - for x in self.__dict__.keys() - if getattr(self, x) is not None and conditions[x] is not None - ] - if not queries: - return None - return and_(*queries) - - @property - def filter_query(self) -> List: - def _base_branch_query(): - if not self.base_branches: - return None - - return or_( - PullRequest.base_branch.op("~")(term) for term in self.base_branches - ) - - def _repo_filters_query(): - if not self.repo_filters: - return None - - return or_( - self.RepoFilter(repo_id, repo_filters).filter_query - for repo_id, repo_filters in self.repo_filters.items() - if repo_filters - ) - - def _excluded_pr_ids_query(): - if not self.excluded_pr_ids: - return None - - return PullRequest.id.notin_(self.excluded_pr_ids) - - def _include_prs_below_max_cycle_time(): - if not self.max_cycle_time: - return None - - return and_( - PullRequest.cycle_time != None, - PullRequest.cycle_time < self.max_cycle_time, - ) - - conditions = { - "base_branches": _base_branch_query(), - "repo_filters": _repo_filters_query(), - "excluded_pr_ids": _excluded_pr_ids_query(), - "max_cycle_time": _include_prs_below_max_cycle_time(), - } - return [ - conditions[x] - for x in self.__dict__.keys() - if getattr(self, x) is not None and conditions[x] is not None - ] diff --git a/apiserver/dora/store/models/code/pull_requests.py b/apiserver/dora/store/models/code/pull_requests.py deleted file mode 100644 index 0be275602..000000000 --- a/apiserver/dora/store/models/code/pull_requests.py +++ /dev/null @@ -1,155 +0,0 @@ -from datetime import datetime - -from sqlalchemy import func -from sqlalchemy.dialects.postgresql import UUID, JSONB, ARRAY, ENUM - -from dora.store import db -from dora.store.models.code.enums import ( - PullRequestEventType, - PullRequestState, - PullRequestRevertPRMappingActorType, -) - - -class PullRequest(db.Model): - __tablename__ = "PullRequest" - - id = db.Column(UUID(as_uuid=True), primary_key=True) - repo_id = db.Column(UUID(as_uuid=True), db.ForeignKey("OrgRepo.id")) - title = db.Column(db.String) - url = db.Column(db.String) - number = db.Column(db.String) - author = db.Column(db.String) - state = db.Column(ENUM(PullRequestState)) - requested_reviews = db.Column(ARRAY(db.String)) - base_branch = db.Column(db.String) - head_branch = db.Column(db.String) - data = db.Column(JSONB) - created_at = db.Column(db.DateTime(timezone=True)) - updated_at = db.Column(db.DateTime(timezone=True)) - state_changed_at = db.Column(db.DateTime(timezone=True)) - first_response_time = db.Column(db.Integer) - rework_time = db.Column(db.Integer) - merge_time = db.Column(db.Integer) - cycle_time = db.Column(db.Integer) - reviewers = db.Column(ARRAY(db.String)) - meta = db.Column(JSONB) - provider = db.Column(db.String) - rework_cycles = db.Column(db.Integer, default=0) - first_commit_to_open = db.Column(db.Integer) - merge_to_deploy = db.Column(db.Integer) - lead_time = db.Column(db.Integer) - merge_commit_sha = db.Column(db.String) - created_in_db_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_in_db_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - def __eq__(self, other): - return self.id == other.id - - def __lt__(self, other): - return self.id < other.id - - def __hash__(self): - return hash(self.id) - - @property - def commits(self) -> int: - return self.meta.get("code_stats", {}).get("commits", 0) - - @property - def additions(self) -> int: - return self.meta.get("code_stats", {}).get("additions", 0) - - @property - def deletions(self) -> int: - return self.meta.get("code_stats", {}).get("deletions", 0) - - @property - def changed_files(self) -> int: - return self.meta.get("code_stats", {}).get("changed_files", 0) - - @property - def comments(self) -> int: - return self.meta.get("code_stats", {}).get("comments", 0) - - @property - def username(self) -> str: - return self.meta.get("user_profile", {}).get("username", "") - - -class PullRequestEvent(db.Model): - __tablename__ = "PullRequestEvent" - - id = db.Column(UUID(as_uuid=True), primary_key=True) - pull_request_id = db.Column(UUID(as_uuid=True), db.ForeignKey("PullRequest.id")) - type = db.Column(ENUM(PullRequestEventType)) - data = db.Column(JSONB) - created_at = db.Column(db.DateTime(timezone=True)) - idempotency_key = db.Column(db.String) - org_repo_id = db.Column(UUID(as_uuid=True), db.ForeignKey("OrgRepo.id")) - actor_username = db.Column(db.String) - created_in_db_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_in_db_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - @property - def state(self): - if self.type in [ - PullRequestEventType.REVIEW.value, - PullRequestEventType.REVIEW, - ]: - return self.data.get("state", "") - - return "" - - -class PullRequestCommit(db.Model): - __tablename__ = "PullRequestCommit" - - hash = db.Column(db.String, primary_key=True) - pull_request_id = db.Column(UUID(as_uuid=True), db.ForeignKey("PullRequest.id")) - message = db.Column(db.String) - url = db.Column(db.String) - data = db.Column(JSONB) - author = db.Column(db.String) - created_at = db.Column(db.DateTime(timezone=True)) - org_repo_id = db.Column(UUID(as_uuid=True), db.ForeignKey("OrgRepo.id")) - created_in_db_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_in_db_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - -class PullRequestRevertPRMapping(db.Model): - __tablename__ = "PullRequestRevertPRMapping" - - pr_id = db.Column( - UUID(as_uuid=True), - db.ForeignKey("PullRequest.id"), - primary_key=True, - nullable=False, - ) - actor_type = db.Column( - ENUM(PullRequestRevertPRMappingActorType), primary_key=True, nullable=False - ) - actor = db.Column(UUID(as_uuid=True), db.ForeignKey("Users.id")) - reverted_pr = db.Column( - UUID(as_uuid=True), db.ForeignKey("PullRequest.id"), nullable=False - ) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - def __hash__(self): - return hash((self.pr_id, self.reverted_pr)) - - def __eq__(self, other): - return ( - isinstance(other, PullRequestRevertPRMapping) - and self.pr_id == other.pr_id - and self.reverted_pr == other.reverted_pr - ) diff --git a/apiserver/dora/store/models/code/repository.py b/apiserver/dora/store/models/code/repository.py deleted file mode 100644 index 12ec95e18..000000000 --- a/apiserver/dora/store/models/code/repository.py +++ /dev/null @@ -1,108 +0,0 @@ -import uuid -from datetime import datetime -from typing import Tuple - -import pytz -from sqlalchemy import func -from sqlalchemy.dialects.postgresql import UUID, JSONB, ARRAY, ENUM - -from dora.store import db -from dora.store.models.code.enums import ( - CodeProvider, - BookmarkType, - TeamReposDeploymentType, -) - - -class OrgRepo(db.Model): - __tablename__ = "OrgRepo" - - id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - org_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Organization.id")) - name = db.Column(db.String) - provider = db.Column(db.String) - org_name = db.Column(db.String) - default_branch = db.Column(db.String) - language = db.Column(db.String) - contributors = db.Column(JSONB) - idempotency_key = db.Column(db.String) - slug = db.Column(db.String) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - is_active = db.Column(db.Boolean, default=True) - - @property - def url(self): - if self.provider == CodeProvider.GITHUB.value: - return f"https://www.github.com/{self.org_name}/{self.name}" - - raise NotImplementedError(f"URL not implemented for {self.provider}") - - @property - def contributor_count(self) -> [Tuple[str, int]]: - if not self.contributors: - return [] - - return self.contributors.get("contributions", []) - - def __hash__(self): - return hash(self.id) - - -class TeamRepos(db.Model): - __tablename__ = "TeamRepos" - - team_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Team.id"), primary_key=True) - org_repo_id = db.Column( - UUID(as_uuid=True), db.ForeignKey("OrgRepo.id"), primary_key=True - ) - prod_branch = db.Column(db.String) - prod_branches = db.Column(ARRAY(db.String)) - deployment_type = db.Column( - ENUM(TeamReposDeploymentType), default=TeamReposDeploymentType.PR_MERGE - ) - is_active = db.Column(db.Boolean, default=True) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - -class RepoSyncLogs(db.Model): - __tablename__ = "RepoSyncLogs" - - repo_id = db.Column( - UUID(as_uuid=True), db.ForeignKey("OrgRepo.id"), primary_key=True - ) - synced_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - - -class Bookmark(db.Model): - __tablename__ = "Bookmark" - - repo_id = db.Column(UUID(as_uuid=True), primary_key=True) - type = db.Column(ENUM(BookmarkType), primary_key=True) - bookmark = db.Column(db.String) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - -class BookmarkMergeToDeployBroker(db.Model): - __tablename__ = "BookmarkMergeToDeployBroker" - - repo_id = db.Column(UUID(as_uuid=True), primary_key=True) - bookmark = db.Column(db.String) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - @property - def bookmark_date(self): - if not self.bookmark: - return None - return datetime.fromisoformat(self.bookmark).astimezone(tz=pytz.UTC) diff --git a/apiserver/dora/store/models/code/workflows/__init__.py b/apiserver/dora/store/models/code/workflows/__init__.py deleted file mode 100644 index 75f95af15..000000000 --- a/apiserver/dora/store/models/code/workflows/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .enums import RepoWorkflowType, RepoWorkflowProviders, RepoWorkflowRunsStatus -from .filter import WorkflowFilter -from .workflows import RepoWorkflow, RepoWorkflowRuns, RepoWorkflowRunsBookmark diff --git a/apiserver/dora/store/models/code/workflows/enums.py b/apiserver/dora/store/models/code/workflows/enums.py deleted file mode 100644 index cfcbd7584..000000000 --- a/apiserver/dora/store/models/code/workflows/enums.py +++ /dev/null @@ -1,32 +0,0 @@ -from enum import Enum - - -class RepoWorkflowProviders(Enum): - GITHUB_ACTIONS = "github" - CIRCLE_CI = "circle_ci" - - @classmethod - def get_workflow_providers(cls): - return [v for v in cls.__members__.values()] - - @classmethod - def get_workflow_providers_values(cls): - return [v.value for v in cls.__members__.values()] - - @classmethod - def get_enum(cls, provider: str): - for v in cls.__members__.values(): - if provider == v.value: - return v - return None - - -class RepoWorkflowType(Enum): - DEPLOYMENT = "DEPLOYMENT" - - -class RepoWorkflowRunsStatus(Enum): - SUCCESS = "SUCCESS" - FAILURE = "FAILURE" - PENDING = "PENDING" - CANCELLED = "CANCELLED" diff --git a/apiserver/dora/store/models/code/workflows/filter.py b/apiserver/dora/store/models/code/workflows/filter.py deleted file mode 100644 index baf220efa..000000000 --- a/apiserver/dora/store/models/code/workflows/filter.py +++ /dev/null @@ -1,81 +0,0 @@ -from dataclasses import dataclass -from operator import and_ -from typing import List, Dict - -from sqlalchemy import or_ - -from .workflows import RepoWorkflowRuns, RepoWorkflow - - -class RepoWorkflowFilter: - def __init__(self, repo_id: str, repo_filters=None): - if repo_filters is None: - repo_filters = {} - self.repo_id = repo_id - self.head_branches = repo_filters.get("head_branches", []) - - @property - def filter_query(self): - def _repo_id_query(): - if not self.repo_id: - raise ValueError("repo_id is required") - return RepoWorkflow.org_repo_id == self.repo_id - - def _head_branches_query(): - if not self.head_branches: - return None - return or_( - RepoWorkflowRuns.head_branch.op("~")(term) - for term in self.head_branches - if term is not None - ) - - conditions = { - "repo_id": _repo_id_query(), - "head_branches": _head_branches_query(), - } - queries = [ - conditions[x] - for x in self.__dict__.keys() - if getattr(self, x) is not None and conditions[x] is not None - ] - if not queries: - return None - return and_(*queries) - - -@dataclass -class WorkflowFilter: - head_branches: List[str] = None - repo_filters: Dict[str, Dict] = None - - @property - def filter_query(self) -> List: - def _head_branches_query(): - if not self.head_branches: - return None - - return or_( - RepoWorkflowRuns.head_branch.op("~")(term) - for term in self.head_branches - ) - - def _repo_filters_query(): - if not self.repo_filters: - return None - - return or_( - RepoWorkflowFilter(repo_id, repo_filters).filter_query - for repo_id, repo_filters in self.repo_filters.items() - if repo_filters - ) - - conditions = { - "head_branches": _head_branches_query(), - "repo_filters": _repo_filters_query(), - } - return [ - conditions[x] - for x in self.__dict__.keys() - if getattr(self, x) is not None and conditions[x] is not None - ] diff --git a/apiserver/dora/store/models/code/workflows/workflows.py b/apiserver/dora/store/models/code/workflows/workflows.py deleted file mode 100644 index e02be6b1f..000000000 --- a/apiserver/dora/store/models/code/workflows/workflows.py +++ /dev/null @@ -1,62 +0,0 @@ -import uuid - -from sqlalchemy import func -from sqlalchemy.dialects.postgresql import UUID, JSONB, ENUM - -from dora.store import db -from dora.store.models.code.workflows.enums import ( - RepoWorkflowType, - RepoWorkflowProviders, - RepoWorkflowRunsStatus, -) - - -class RepoWorkflow(db.Model): - __tablename__ = "RepoWorkflow" - - id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - org_repo_id = db.Column(UUID(as_uuid=True), db.ForeignKey("OrgRepo.id")) - type = db.Column(ENUM(RepoWorkflowType)) - provider = db.Column(ENUM(RepoWorkflowProviders)) - provider_workflow_id = db.Column(db.String, nullable=False) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - meta = db.Column(JSONB, default="{}") - is_active = db.Column(db.Boolean, default=True) - name = db.Column(db.String) - - -class RepoWorkflowRuns(db.Model): - __tablename__ = "RepoWorkflowRuns" - - id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - repo_workflow_id = db.Column(UUID(as_uuid=True), db.ForeignKey("RepoWorkflow.id")) - provider_workflow_run_id = db.Column(db.String, nullable=False) - event_actor = db.Column(db.String) - head_branch = db.Column(db.String) - status = db.Column(ENUM(RepoWorkflowRunsStatus)) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - conducted_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - meta = db.Column(JSONB, default="{}") - duration = db.Column(db.Integer) - html_url = db.Column(db.String) - - def __hash__(self): - return hash(self.id) - - -class RepoWorkflowRunsBookmark(db.Model): - __tablename__ = "RepoWorkflowRunsBookmark" - - id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - repo_workflow_id = db.Column(UUID(as_uuid=True), db.ForeignKey("RepoWorkflow.id")) - bookmark = db.Column(db.String) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) diff --git a/apiserver/dora/store/models/core/__init__.py b/apiserver/dora/store/models/core/__init__.py deleted file mode 100644 index 6c21e1bb6..000000000 --- a/apiserver/dora/store/models/core/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .organization import Organization -from .teams import Team -from .users import Users diff --git a/apiserver/dora/store/models/core/organization.py b/apiserver/dora/store/models/core/organization.py deleted file mode 100644 index c275de442..000000000 --- a/apiserver/dora/store/models/core/organization.py +++ /dev/null @@ -1,23 +0,0 @@ -from sqlalchemy.dialects.postgresql import UUID, ARRAY - -from dora.store import db - - -class Organization(db.Model): - __tablename__ = "Organization" - - id = db.Column(UUID(as_uuid=True), primary_key=True) - name = db.Column(db.String) - created_at = db.Column(db.DateTime(timezone=True)) - domain = db.Column(db.String) - other_domains = db.Column(ARRAY(db.String)) - - def __eq__(self, other): - - if isinstance(other, Organization): - return self.id == other.id - - return False - - def __hash__(self): - return hash(self.id) diff --git a/apiserver/dora/store/models/core/teams.py b/apiserver/dora/store/models/core/teams.py deleted file mode 100644 index 13884113b..000000000 --- a/apiserver/dora/store/models/core/teams.py +++ /dev/null @@ -1,26 +0,0 @@ -import uuid - -from sqlalchemy import ( - func, -) -from sqlalchemy.dialects.postgresql import UUID, ARRAY - -from dora.store import db - - -class Team(db.Model): - __tablename__ = "Team" - - id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - org_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Organization.id")) - name = db.Column(db.String) - member_ids = db.Column(ARRAY(UUID(as_uuid=True)), nullable=False) - manager_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Users.id")) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - is_deleted = db.Column(db.Boolean, default=False) - - def __hash__(self): - return hash(self.id) diff --git a/apiserver/dora/store/models/core/users.py b/apiserver/dora/store/models/core/users.py deleted file mode 100644 index e34e005b8..000000000 --- a/apiserver/dora/store/models/core/users.py +++ /dev/null @@ -1,21 +0,0 @@ -from sqlalchemy import ( - func, -) -from sqlalchemy.dialects.postgresql import UUID - -from dora.store import db - - -class Users(db.Model): - __tablename__ = "Users" - - id = db.Column(UUID(as_uuid=True), primary_key=True) - org_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Organization.id")) - name = db.Column(db.String) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - primary_email = db.Column(db.String) - is_deleted = db.Column(db.Boolean, default=False) - avatar_url = db.Column(db.String) diff --git a/apiserver/dora/store/models/incidents/__init__.py b/apiserver/dora/store/models/incidents/__init__.py deleted file mode 100644 index aef271901..000000000 --- a/apiserver/dora/store/models/incidents/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from .enums import ( - IncidentType, - IncidentBookmarkType, - IncidentProvider, - ServiceStatus, - IncidentStatus, - IncidentSource, -) -from .filter import IncidentFilter -from .incidents import ( - Incident, - IncidentOrgIncidentServiceMap, - IncidentsBookmark, -) -from .services import OrgIncidentService, TeamIncidentService diff --git a/apiserver/dora/store/models/incidents/enums.py b/apiserver/dora/store/models/incidents/enums.py deleted file mode 100644 index 932945f8a..000000000 --- a/apiserver/dora/store/models/incidents/enums.py +++ /dev/null @@ -1,35 +0,0 @@ -from enum import Enum - - -class IncidentProvider(Enum): - GITHUB = "github" - - -class IncidentSource(Enum): - INCIDENT_SERVICE = "INCIDENT_SERVICE" - INCIDENT_TEAM = "INCIDENT_TEAM" - GIT_REPO = "GIT_REPO" - - -class ServiceStatus(Enum): - DISABLED = "disabled" - ACTIVE = "active" - WARNING = "warning" - CRITICAL = "critical" - MAINTENANCE = "maintenance" - - -class IncidentStatus(Enum): - TRIGGERED = "triggered" - ACKNOWLEDGED = "acknowledged" - RESOLVED = "resolved" - - -class IncidentType(Enum): - INCIDENT = "INCIDENT" - REVERT_PR = "REVERT_PR" - ALERT = "ALERT" - - -class IncidentBookmarkType(Enum): - SERVICE = "SERVICE" diff --git a/apiserver/dora/store/models/incidents/filter.py b/apiserver/dora/store/models/incidents/filter.py deleted file mode 100644 index d041c1f1a..000000000 --- a/apiserver/dora/store/models/incidents/filter.py +++ /dev/null @@ -1,45 +0,0 @@ -from dataclasses import dataclass -from typing import List - -from sqlalchemy import or_ - -from dora.store.models.incidents.incidents import Incident - - -@dataclass -class IncidentFilter: - """Dataclass for filtering incidents.""" - - title_filter_substrings: List[str] = None - incident_types: List[str] = None - - @property - def filter_query(self) -> List: - def _title_filter_substrings_query(): - if not self.title_filter_substrings: - return None - - return or_( - Incident.title.contains(substring, autoescape=True) - for substring in self.title_filter_substrings - ) - - def _incident_type_query(): - if not self.incident_types: - return None - - return or_( - Incident.incident_type == incident_type - for incident_type in self.incident_types - ) - - conditions = { - "title_filter_substrings": _title_filter_substrings_query(), - "incident_types": _incident_type_query(), - } - - return [ - conditions[x] - for x in self.__dict__.keys() - if getattr(self, x) is not None and conditions[x] is not None - ] diff --git a/apiserver/dora/store/models/incidents/incidents.py b/apiserver/dora/store/models/incidents/incidents.py deleted file mode 100644 index 464744e0c..000000000 --- a/apiserver/dora/store/models/incidents/incidents.py +++ /dev/null @@ -1,59 +0,0 @@ -from sqlalchemy import ( - func, -) -from sqlalchemy.dialects.postgresql import UUID, ARRAY, JSONB, ENUM - -from dora.store import db -from dora.store.models.incidents.enums import IncidentType, IncidentBookmarkType - - -class Incident(db.Model): - __tablename__ = "Incident" - - id = db.Column(UUID(as_uuid=True), primary_key=True) - provider = db.Column(db.String) - key = db.Column(db.String) - incident_number = db.Column(db.Integer) - title = db.Column(db.String) - status = db.Column(db.String) - creation_date = db.Column(db.DateTime(timezone=True)) - acknowledged_date = db.Column(db.DateTime(timezone=True)) - resolved_date = db.Column(db.DateTime(timezone=True)) - assigned_to = db.Column(db.String) - assignees = db.Column(ARRAY(db.String)) - incident_type = db.Column(ENUM(IncidentType), default=IncidentType.INCIDENT) - meta = db.Column(JSONB, default={}) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - def __hash__(self): - return hash(self.id) - - -class IncidentOrgIncidentServiceMap(db.Model): - __tablename__ = "IncidentOrgIncidentServiceMap" - - incident_id = db.Column( - UUID(as_uuid=True), db.ForeignKey("Incident.id"), primary_key=True - ) - service_id = db.Column( - UUID(as_uuid=True), db.ForeignKey("OrgIncidentService.id"), primary_key=True - ) - - -class IncidentsBookmark(db.Model): - __tablename__ = "IncidentsBookmark" - - id = db.Column(UUID(as_uuid=True), primary_key=True) - provider = db.Column(db.String) - entity_id = db.Column(UUID(as_uuid=True)) - entity_type = db.Column( - ENUM(IncidentBookmarkType), default=IncidentBookmarkType.SERVICE - ) - bookmark = db.Column(db.DateTime(timezone=True), server_default=func.now()) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) diff --git a/apiserver/dora/store/models/incidents/services.py b/apiserver/dora/store/models/incidents/services.py deleted file mode 100644 index 14b487c32..000000000 --- a/apiserver/dora/store/models/incidents/services.py +++ /dev/null @@ -1,45 +0,0 @@ -from sqlalchemy import ( - func, -) -from sqlalchemy.dialects.postgresql import UUID, ARRAY, JSONB, ENUM -from sqlalchemy.orm import relationship - -from dora.store import db -from dora.store.models.incidents import IncidentSource - - -class OrgIncidentService(db.Model): - __tablename__ = "OrgIncidentService" - - id = db.Column(UUID(as_uuid=True), primary_key=True) - org_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Organization.id")) - name = db.Column(db.String) - provider = db.Column(db.String) - key = db.Column(db.String) - auto_resolve_timeout = db.Column(db.Integer) - acknowledgement_timeout = db.Column(db.Integer) - created_by = db.Column(db.String) - provider_team_keys = db.Column(ARRAY(db.String)) - status = db.Column(db.String) - is_deleted = db.Column(db.Boolean, default=False) - meta = db.Column(JSONB, default={}) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - source_type = db.Column( - ENUM(IncidentSource), default=IncidentSource.INCIDENT_SERVICE, nullable=False - ) - - -class TeamIncidentService(db.Model): - __tablename__ = "TeamIncidentService" - - id = db.Column(UUID(as_uuid=True), primary_key=True) - team_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Team.id")) - service_id = db.Column(UUID(as_uuid=True), db.ForeignKey("OrgIncidentService.id")) - OrgIncidentService = relationship("OrgIncidentService", lazy="joined") - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) diff --git a/apiserver/dora/store/models/integrations/__init__.py b/apiserver/dora/store/models/integrations/__init__.py deleted file mode 100644 index 0192a8591..000000000 --- a/apiserver/dora/store/models/integrations/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .enums import UserIdentityProvider -from .integrations import Integration, UserIdentity diff --git a/apiserver/dora/store/models/integrations/enums.py b/apiserver/dora/store/models/integrations/enums.py deleted file mode 100644 index 6f4e8b449..000000000 --- a/apiserver/dora/store/models/integrations/enums.py +++ /dev/null @@ -1,12 +0,0 @@ -from enum import Enum - - -class UserIdentityProvider(Enum): - GITHUB = "github" - - @classmethod - def get_enum(self, provider: str): - for v in self.__members__.values(): - if provider == v.value: - return v - return None diff --git a/apiserver/dora/store/models/integrations/integrations.py b/apiserver/dora/store/models/integrations/integrations.py deleted file mode 100644 index c0010e3f4..000000000 --- a/apiserver/dora/store/models/integrations/integrations.py +++ /dev/null @@ -1,49 +0,0 @@ -from sqlalchemy import ( - func, -) -from sqlalchemy.dialects.postgresql import UUID, ARRAY, JSONB - -from dora.store import db -from dora.store.models.integrations import UserIdentityProvider - - -class Integration(db.Model): - __tablename__ = "Integration" - - org_id = db.Column( - UUID(as_uuid=True), db.ForeignKey("Organization.id"), primary_key=True - ) - name = db.Column(db.String, primary_key=True) - generated_by = db.Column( - UUID(as_uuid=True), db.ForeignKey("Users.id"), nullable=True - ) - access_token_enc_chunks = db.Column(ARRAY(db.String)) - refresh_token_enc_chunks = db.Column(ARRAY(db.String)) - provider_meta = db.Column(JSONB) - scopes = db.Column(ARRAY(db.String)) - access_token_valid_till = db.Column(db.DateTime(timezone=True)) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - -class UserIdentity(db.Model): - __tablename__ = "UserIdentity" - - user_id = db.Column(UUID(as_uuid=True), primary_key=True) - provider = db.Column(db.String, primary_key=True) - token = db.Column(db.String) - username = db.Column(db.String) - refresh_token = db.Column(db.String) - org_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Organization.id")) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - meta = db.Column(JSONB) - - @property - def avatar_url(self): - if self.provider == UserIdentityProvider.GITHUB.value: - return f"https://github.com/{self.username}.png" diff --git a/apiserver/dora/store/models/settings/__init__.py b/apiserver/dora/store/models/settings/__init__.py deleted file mode 100644 index 450b0adce..000000000 --- a/apiserver/dora/store/models/settings/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .configuration_settings import SettingType, Settings -from .enums import EntityType diff --git a/apiserver/dora/store/models/settings/configuration_settings.py b/apiserver/dora/store/models/settings/configuration_settings.py deleted file mode 100644 index e5613ee11..000000000 --- a/apiserver/dora/store/models/settings/configuration_settings.py +++ /dev/null @@ -1,33 +0,0 @@ -from enum import Enum - -from sqlalchemy import func -from sqlalchemy.dialects.postgresql import UUID, ENUM, JSONB - -from dora.store import db -from dora.store.models.settings.enums import EntityType - -""" -All Data config settings will be stored in the below table. -""" - - -class SettingType(Enum): - INCIDENT_SETTING = "INCIDENT_SETTING" - INCIDENT_TYPES_SETTING = "INCIDENT_TYPES_SETTING" - INCIDENT_SOURCES_SETTING = "INCIDENT_SOURCES_SETTING" - EXCLUDED_PRS_SETTING = "EXCLUDED_PRS_SETTING" - - -class Settings(db.Model): - __tablename__ = "Settings" - - entity_id = db.Column(UUID(as_uuid=True), primary_key=True, nullable=False) - entity_type = db.Column(ENUM(EntityType), primary_key=True, nullable=False) - setting_type = db.Column(ENUM(SettingType), primary_key=True, nullable=False) - updated_by = db.Column(UUID(as_uuid=True), db.ForeignKey("Users.id")) - created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) - updated_at = db.Column( - db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - data = db.Column(JSONB, default="{}") - is_deleted = db.Column(db.Boolean, default=False) diff --git a/apiserver/dora/store/models/settings/enums.py b/apiserver/dora/store/models/settings/enums.py deleted file mode 100644 index caa351ebe..000000000 --- a/apiserver/dora/store/models/settings/enums.py +++ /dev/null @@ -1,7 +0,0 @@ -from enum import Enum - - -class EntityType(Enum): - USER = "USER" - TEAM = "TEAM" - ORG = "ORG" diff --git a/apiserver/dora/store/repos/__init__.py b/apiserver/dora/store/repos/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/dora/store/repos/code.py b/apiserver/dora/store/repos/code.py deleted file mode 100644 index 512d303d5..000000000 --- a/apiserver/dora/store/repos/code.py +++ /dev/null @@ -1,333 +0,0 @@ -from datetime import datetime -from operator import and_ -from typing import Optional, List - -from sqlalchemy import or_ -from sqlalchemy.orm import defer - -from dora.store import db, rollback_on_exc -from dora.store.models.code import ( - PullRequest, - PullRequestEvent, - OrgRepo, - PullRequestRevertPRMapping, - PullRequestCommit, - Bookmark, - TeamRepos, - PullRequestState, - PRFilter, - BookmarkMergeToDeployBroker, -) -from dora.utils.time import Interval - - -class CodeRepoService: - def __init__(self): - self._db = db - - @rollback_on_exc - def get_active_org_repos(self, org_id: str) -> List[OrgRepo]: - return ( - self._db.session.query(OrgRepo) - .filter(OrgRepo.org_id == org_id, OrgRepo.is_active.is_(True)) - .all() - ) - - @rollback_on_exc - def update_org_repos(self, org_repos: List[OrgRepo]): - [self._db.session.merge(org_repo) for org_repo in org_repos] - self._db.session.commit() - - @rollback_on_exc - def save_pull_requests_data( - self, - pull_requests: List[PullRequest], - pull_request_commits: List[PullRequestCommit], - pull_request_events: List[PullRequestEvent], - ): - [self._db.session.merge(pull_request) for pull_request in pull_requests] - [ - self._db.session.merge(pull_request_commit) - for pull_request_commit in pull_request_commits - ] - [ - self._db.session.merge(pull_request_event) - for pull_request_event in pull_request_events - ] - self._db.session.commit() - - @rollback_on_exc - def update_prs(self, prs: List[PullRequest]): - [self._db.session.merge(pr) for pr in prs] - self._db.session.commit() - - @rollback_on_exc - def save_revert_pr_mappings( - self, revert_pr_mappings: List[PullRequestRevertPRMapping] - ): - [self._db.session.merge(revert_pr_map) for revert_pr_map in revert_pr_mappings] - self._db.session.commit() - - @rollback_on_exc - def get_org_repo_bookmark(self, org_repo: OrgRepo, bookmark_type): - return ( - self._db.session.query(Bookmark) - .filter( - and_( - Bookmark.repo_id == org_repo.id, - Bookmark.type == bookmark_type.value, - ) - ) - .one_or_none() - ) - - @rollback_on_exc - def update_org_repo_bookmark(self, bookmark: Bookmark): - self._db.session.merge(bookmark) - self._db.session.commit() - - @rollback_on_exc - def get_repo_by_id(self, repo_id: str) -> Optional[OrgRepo]: - return ( - self._db.session.query(OrgRepo).filter(OrgRepo.id == repo_id).one_or_none() - ) - - @rollback_on_exc - def get_repo_pr_by_number(self, repo_id: str, pr_number) -> Optional[PullRequest]: - return ( - self._db.session.query(PullRequest) - .options(defer(PullRequest.data)) - .filter( - and_( - PullRequest.repo_id == repo_id, PullRequest.number == str(pr_number) - ) - ) - .one_or_none() - ) - - @rollback_on_exc - def get_pr_events(self, pr_model: PullRequest): - if not pr_model: - return [] - - pr_events = ( - self._db.session.query(PullRequestEvent) - .options(defer(PullRequestEvent.data)) - .filter(PullRequestEvent.pull_request_id == pr_model.id) - .all() - ) - return pr_events - - @rollback_on_exc - def get_prs_by_ids(self, pr_ids: List[str]): - query = ( - self._db.session.query(PullRequest) - .options(defer(PullRequest.data)) - .filter(PullRequest.id.in_(pr_ids)) - ) - return query.all() - - @rollback_on_exc - def get_prs_by_head_branch_match_strings( - self, repo_ids: List[str], match_strings: List[str] - ) -> List[PullRequest]: - query = ( - self._db.session.query(PullRequest) - .options(defer(PullRequest.data)) - .filter( - and_( - PullRequest.repo_id.in_(repo_ids), - or_( - *[ - PullRequest.head_branch.ilike(f"{match_string}%") - for match_string in match_strings - ] - ), - ) - ) - .order_by(PullRequest.updated_in_db_at.desc()) - ) - - return query.all() - - @rollback_on_exc - def get_reverted_prs_by_numbers( - self, repo_ids: List[str], numbers: List[str] - ) -> List[PullRequest]: - query = ( - self._db.session.query(PullRequest) - .options(defer(PullRequest.data)) - .filter( - and_( - PullRequest.repo_id.in_(repo_ids), - PullRequest.number.in_(numbers), - ) - ) - .order_by(PullRequest.updated_in_db_at.desc()) - ) - - return query.all() - - @rollback_on_exc - def get_active_team_repos_by_team_id(self, team_id: str) -> List[TeamRepos]: - return ( - self._db.session.query(TeamRepos) - .filter(TeamRepos.team_id == team_id, TeamRepos.is_active.is_(True)) - .all() - ) - - @rollback_on_exc - def get_active_team_repos_by_team_ids(self, team_ids: List[str]) -> List[TeamRepos]: - return ( - self._db.session.query(TeamRepos) - .filter(TeamRepos.team_id.in_(team_ids), TeamRepos.is_active.is_(True)) - .all() - ) - - @rollback_on_exc - def get_active_org_repos_by_ids(self, repo_ids: List[str]) -> List[OrgRepo]: - return ( - self._db.session.query(OrgRepo) - .filter(OrgRepo.id.in_(repo_ids), OrgRepo.is_active.is_(True)) - .all() - ) - - @rollback_on_exc - def get_prs_merged_in_interval( - self, - repo_ids: List[str], - interval: Interval, - pr_filter: PRFilter = None, - base_branches: List[str] = None, - has_non_null_mtd=False, - ) -> List[PullRequest]: - query = self._db.session.query(PullRequest).options(defer(PullRequest.data)) - - query = self._filter_prs_by_repo_ids(query, repo_ids) - query = self._filter_prs_merged_in_interval(query, interval) - - query = self._filter_prs(query, pr_filter) - query = self._filter_base_branch_on_regex(query, base_branches) - - if has_non_null_mtd: - query = query.filter(PullRequest.merge_to_deploy.is_not(None)) - - query = query.order_by(PullRequest.state_changed_at.asc()) - - return query.all() - - @rollback_on_exc - def get_pull_request_by_id(self, pr_id: str) -> PullRequest: - return ( - self._db.session.query(PullRequest) - .options(defer(PullRequest.data)) - .filter(PullRequest.id == pr_id) - .one_or_none() - ) - - @rollback_on_exc - def get_previous_pull_request(self, pull_request: PullRequest) -> PullRequest: - return ( - self._db.session.query(PullRequest) - .options(defer(PullRequest.data)) - .filter( - PullRequest.repo_id == pull_request.repo_id, - PullRequest.state_changed_at < pull_request.state_changed_at, - PullRequest.base_branch == pull_request.base_branch, - PullRequest.state == PullRequestState.MERGED, - ) - .order_by(PullRequest.state_changed_at.desc()) - .first() - ) - - @rollback_on_exc - def get_repos_by_ids(self, ids: List[str]) -> List[OrgRepo]: - if not ids: - return [] - - return self._db.session.query(OrgRepo).filter(OrgRepo.id.in_(ids)).all() - - @rollback_on_exc - def get_team_repos(self, team_id) -> List[OrgRepo]: - team_repos = ( - self._db.session.query(TeamRepos) - .filter(and_(TeamRepos.team_id == team_id, TeamRepos.is_active == True)) - .all() - ) - if not team_repos: - return [] - - team_repo_ids = [tr.org_repo_id for tr in team_repos] - return self.get_repos_by_ids(team_repo_ids) - - @rollback_on_exc - def get_merge_to_deploy_broker_bookmark( - self, repo_id: str - ) -> BookmarkMergeToDeployBroker: - return ( - self._db.session.query(BookmarkMergeToDeployBroker) - .filter(BookmarkMergeToDeployBroker.repo_id == repo_id) - .one_or_none() - ) - - @rollback_on_exc - def update_merge_to_deploy_broker_bookmark( - self, bookmark: BookmarkMergeToDeployBroker - ): - self._db.session.merge(bookmark) - self._db.session.commit() - - @rollback_on_exc - def get_prs_in_repo_merged_before_given_date_with_merge_to_deploy_as_null( - self, repo_id: str, to_time: datetime - ): - return ( - self._db.session.query(PullRequest) - .options(defer(PullRequest.data)) - .filter( - PullRequest.repo_id == repo_id, - PullRequest.state == PullRequestState.MERGED, - PullRequest.state_changed_at <= to_time, - PullRequest.merge_to_deploy.is_(None), - ) - .all() - ) - - @rollback_on_exc - def get_repo_revert_prs_mappings_updated_in_interval( - self, repo_id, from_time, to_time - ) -> List[PullRequestRevertPRMapping]: - query = ( - self._db.session.query(PullRequestRevertPRMapping) - .join(PullRequest, PullRequest.id == PullRequestRevertPRMapping.pr_id) - .filter( - PullRequest.repo_id == repo_id, - PullRequest.state == PullRequestState.MERGED, - PullRequestRevertPRMapping.updated_at.between(from_time, to_time), - ) - ) - query = query.order_by(PullRequest.updated_at.desc()) - - return query.all() - - def _filter_prs_by_repo_ids(self, query, repo_ids: List[str]): - return query.filter(PullRequest.repo_id.in_(repo_ids)) - - def _filter_prs_merged_in_interval(self, query, interval: Interval): - return query.filter( - PullRequest.state_changed_at.between(interval.from_time, interval.to_time), - PullRequest.state == PullRequestState.MERGED, - ) - - def _filter_prs(self, query, pr_filter: PRFilter): - if pr_filter: - query = query.filter(*pr_filter.filter_query) - return query - - def _filter_base_branch_on_regex(self, query, base_branches: List[str] = None): - if base_branches: - conditions = [ - PullRequest.base_branch.op("~")(term) for term in base_branches - ] - return query.filter(or_(*conditions)) - return query diff --git a/apiserver/dora/store/repos/core.py b/apiserver/dora/store/repos/core.py deleted file mode 100644 index 14d3d0fe5..000000000 --- a/apiserver/dora/store/repos/core.py +++ /dev/null @@ -1,108 +0,0 @@ -from typing import Optional, List - -from sqlalchemy import and_ - -from dora.store import db, rollback_on_exc -from dora.store.models import UserIdentityProvider, Integration -from dora.store.models.core import Organization, Team, Users -from dora.utils.cryptography import get_crypto_service - - -class CoreRepoService: - def __init__(self): - self._crypto = get_crypto_service() - self._db = db - - @rollback_on_exc - def get_org(self, org_id): - return ( - self._db.session.query(Organization) - .filter(Organization.id == org_id) - .one_or_none() - ) - - @rollback_on_exc - def get_org_by_name(self, org_name: str): - return ( - self._db.session.query(Organization) - .filter(Organization.name == org_name) - .one_or_none() - ) - - @rollback_on_exc - def get_team(self, team_id: str) -> Team: - return ( - self._db.session.query(Team) - .filter(Team.id == team_id, Team.is_deleted.is_(False)) - .one_or_none() - ) - - @rollback_on_exc - def delete_team(self, team_id: str): - - team = self._db.session.query(Team).filter(Team.id == team_id).one_or_none() - - if not team: - return None - - team.is_deleted = True - - self._db.session.merge(team) - self._db.session.commit() - return self._db.session.query(Team).filter(Team.id == team_id).one_or_none() - - @rollback_on_exc - def create_team(self, org_id: str, name: str, member_ids: List[str]) -> Team: - team = Team( - name=name, - org_id=org_id, - member_ids=member_ids or [], - is_deleted=False, - ) - self._db.session.add(team) - self._db.session.commit() - - return self.get_team(team.id) - - @rollback_on_exc - def update_team(self, team: Team) -> Team: - self._db.session.merge(team) - self._db.session.commit() - - return self.get_team(team.id) - - @rollback_on_exc - def get_user(self, user_id) -> Optional[Users]: - return self._db.session.query(Users).filter(Users.id == user_id).one_or_none() - - @rollback_on_exc - def get_users(self, user_ids: List[str]) -> List[Users]: - return ( - self._db.session.query(Users) - .filter(and_(Users.id.in_(user_ids), Users.is_deleted == False)) - .all() - ) - - @rollback_on_exc - def get_org_integrations_for_names(self, org_id: str, provider_names: List[str]): - return ( - self._db.session.query(Integration) - .filter( - and_(Integration.org_id == org_id, Integration.name.in_(provider_names)) - ) - .all() - ) - - @rollback_on_exc - def get_access_token(self, org_id, provider: UserIdentityProvider) -> Optional[str]: - user_identity: Integration = ( - self._db.session.query(Integration) - .filter( - and_(Integration.org_id == org_id, Integration.name == provider.value) - ) - .one_or_none() - ) - - if not user_identity or not user_identity.access_token_enc_chunks: - return None - return self._crypto.decrypt_chunks(user_identity.access_token_enc_chunks) diff --git a/apiserver/dora/store/repos/incidents.py b/apiserver/dora/store/repos/incidents.py deleted file mode 100644 index 532fa8c99..000000000 --- a/apiserver/dora/store/repos/incidents.py +++ /dev/null @@ -1,148 +0,0 @@ -from typing import List - -from sqlalchemy import and_ - -from dora.store import db, rollback_on_exc -from dora.store.models.incidents import ( - Incident, - IncidentFilter, - IncidentOrgIncidentServiceMap, - TeamIncidentService, - IncidentStatus, - IncidentType, - IncidentProvider, - OrgIncidentService, - IncidentsBookmark, - IncidentBookmarkType, -) -from dora.utils.time import Interval - - -class IncidentsRepoService: - def __init__(self): - self._db = db - - @rollback_on_exc - def get_org_incident_services(self, org_id: str) -> List[OrgIncidentService]: - return ( - self._db.session.query(OrgIncidentService) - .filter(OrgIncidentService.org_id == org_id) - .all() - ) - - @rollback_on_exc - def update_org_incident_services(self, incident_services: List[OrgIncidentService]): - [ - self._db.session.merge(incident_service) - for incident_service in incident_services - ] - self._db.session.commit() - - @rollback_on_exc - def get_incidents_bookmark( - self, - entity_id: str, - entity_type: IncidentBookmarkType, - provider: IncidentProvider, - ) -> IncidentsBookmark: - return ( - self._db.session.query(IncidentsBookmark) - .filter( - and_( - IncidentsBookmark.entity_id == entity_id, - IncidentsBookmark.entity_type == entity_type, - IncidentsBookmark.provider == provider.value, - ) - ) - .one_or_none() - ) - - @rollback_on_exc - def save_incidents_bookmark(self, bookmark: IncidentsBookmark): - self._db.session.merge(bookmark) - self._db.session.commit() - - @rollback_on_exc - def save_incidents_data( - self, - incidents: List[Incident], - incident_org_incident_service_map: List[IncidentOrgIncidentServiceMap], - ): - [self._db.session.merge(incident) for incident in incidents] - [ - self._db.session.merge(incident_service_map) - for incident_service_map in incident_org_incident_service_map - ] - self._db.session.commit() - - @rollback_on_exc - def get_resolved_team_incidents( - self, team_id: str, interval: Interval, incident_filter: IncidentFilter = None - ) -> List[Incident]: - query = self._get_team_incidents_query(team_id, incident_filter) - - query = query.filter( - and_( - Incident.status == IncidentStatus.RESOLVED.value, - Incident.resolved_date.between(interval.from_time, interval.to_time), - ) - ) - - return query.all() - - @rollback_on_exc - def get_team_incidents( - self, team_id: str, interval: Interval, incident_filter: IncidentFilter = None - ) -> List[Incident]: - query = self._get_team_incidents_query(team_id, incident_filter) - - query = query.filter( - Incident.creation_date.between(interval.from_time, interval.to_time), - ) - - return query.all() - - @rollback_on_exc - def get_incident_by_key_type_and_provider( - self, key: str, incident_type: IncidentType, provider: IncidentProvider - ) -> Incident: - return ( - self._db.session.query(Incident) - .filter( - and_( - Incident.key == key, - Incident.incident_type == incident_type, - Incident.provider == provider.value, - ) - ) - .one_or_none() - ) - - def _get_team_incidents_query( - self, team_id: str, incident_filter: IncidentFilter = None - ): - query = ( - self._db.session.query(Incident) - .join( - IncidentOrgIncidentServiceMap, - Incident.id == IncidentOrgIncidentServiceMap.incident_id, - ) - .join( - TeamIncidentService, - IncidentOrgIncidentServiceMap.service_id - == TeamIncidentService.service_id, - ) - .filter( - TeamIncidentService.team_id == team_id, - ) - ) - - query = self._apply_incident_filter(query, incident_filter) - - return query.order_by(Incident.creation_date.asc()) - - def _apply_incident_filter(self, query, incident_filter: IncidentFilter = None): - if not incident_filter: - return query - query = query.filter(*incident_filter.filter_query) - return query diff --git a/apiserver/dora/store/repos/integrations.py b/apiserver/dora/store/repos/integrations.py deleted file mode 100644 index 6ca35ce41..000000000 --- a/apiserver/dora/store/repos/integrations.py +++ /dev/null @@ -1,2 +0,0 @@ -class IntegrationsRepoService: - pass diff --git a/apiserver/dora/store/repos/settings.py b/apiserver/dora/store/repos/settings.py deleted file mode 100644 index 24736d004..000000000 --- a/apiserver/dora/store/repos/settings.py +++ /dev/null @@ -1,90 +0,0 @@ -from typing import Optional, List - -from sqlalchemy import and_ - -from dora.store import db, rollback_on_exc -from dora.store.models import ( - Settings, - SettingType, - EntityType, - Users, -) -from dora.utils.time import time_now - - -class SettingsRepoService: - def __init__(self): - self._db = db - - @rollback_on_exc - def get_setting( - self, entity_id: str, entity_type: EntityType, setting_type: SettingType - ) -> Optional[Settings]: - return ( - self._db.session.query(Settings) - .filter( - and_( - Settings.setting_type == setting_type, - Settings.entity_type == entity_type, - Settings.entity_id == entity_id, - Settings.is_deleted == False, - ) - ) - .one_or_none() - ) - - @rollback_on_exc - def create_settings(self, settings: List[Settings]) -> List[Settings]: - [self._db.session.merge(setting) for setting in settings] - self._db.session.commit() - return settings - - @rollback_on_exc - def save_setting(self, setting: Settings) -> Optional[Settings]: - self._db.session.merge(setting) - self._db.session.commit() - - return self.get_setting( - entity_id=setting.entity_id, - entity_type=setting.entity_type, - setting_type=setting.setting_type, - ) - - @rollback_on_exc - def delete_setting( - self, - entity_id: str, - entity_type: EntityType, - setting_type: SettingType, - deleted_by: Users, - ) -> Optional[Settings]: - setting = self.get_setting(entity_id, entity_type, setting_type) - if not setting: - return - - setting.is_deleted = True - setting.updated_by = deleted_by.id - setting.updated_at = time_now() - self._db.session.merge(setting) - self._db.session.commit() - return setting - - @rollback_on_exc - def get_settings( - self, - entity_id: str, - entity_type: EntityType, - setting_types: List[SettingType], - ) -> Optional[Settings]: - return ( - self._db.session.query(Settings) - .filter( - and_( - Settings.setting_type.in_(setting_types), - Settings.entity_type == entity_type, - Settings.entity_id == entity_id, - Settings.is_deleted == False, - ) - ) - .all() - ) diff --git a/apiserver/dora/store/repos/workflows.py b/apiserver/dora/store/repos/workflows.py deleted file mode 100644 index 8b8fefa23..000000000 --- a/apiserver/dora/store/repos/workflows.py +++ /dev/null @@ -1,227 +0,0 @@ -from datetime import datetime -from typing import List, Tuple - -from sqlalchemy.orm import defer -from sqlalchemy import and_ - -from dora.store import db, rollback_on_exc -from dora.store.models.code.workflows.enums import ( - RepoWorkflowRunsStatus, - RepoWorkflowType, - RepoWorkflowProviders, -) -from dora.store.models.code.workflows.filter import WorkflowFilter -from dora.store.models.code.workflows.workflows import ( - RepoWorkflow, - RepoWorkflowRuns, - RepoWorkflowRunsBookmark, -) -from dora.utils.time import Interval - - -class WorkflowRepoService: - def __init__(self): - self._db = db - - @rollback_on_exc - def get_active_repo_workflows_by_repo_ids_and_providers( - self, repo_ids: List[str], providers: List[RepoWorkflowProviders] - ) -> List[RepoWorkflow]: - - return ( - self._db.session.query(RepoWorkflow) - .options(defer(RepoWorkflow.meta)) - .filter( - RepoWorkflow.org_repo_id.in_(repo_ids), - RepoWorkflow.provider.in_(providers), - RepoWorkflow.is_active.is_(True), - ) - .all() - ) - - @rollback_on_exc - def get_repo_workflow_run_by_provider_workflow_run_id( - self, repo_workflow_id: str, provider_workflow_run_id: str - ) -> RepoWorkflowRuns: - return ( - self._db.session.query(RepoWorkflowRuns) - .filter( - RepoWorkflowRuns.repo_workflow_id == repo_workflow_id, - RepoWorkflowRuns.provider_workflow_run_id == provider_workflow_run_id, - ) - .one_or_none() - ) - - @rollback_on_exc - def save_repo_workflow_runs(self, repo_workflow_runs: List[RepoWorkflowRuns]): - [ - self._db.session.merge(repo_workflow_run) - for repo_workflow_run in repo_workflow_runs - ] - self._db.session.commit() - - @rollback_on_exc - def get_repo_workflow_runs_bookmark( - self, repo_workflow_id: str - ) -> RepoWorkflowRunsBookmark: - return ( - self._db.session.query(RepoWorkflowRunsBookmark) - .filter(RepoWorkflowRunsBookmark.repo_workflow_id == repo_workflow_id) - .one_or_none() - ) - - @rollback_on_exc - def update_repo_workflow_runs_bookmark(self, bookmark: RepoWorkflowRunsBookmark): - self._db.session.merge(bookmark) - self._db.session.commit() - - @rollback_on_exc - def get_repo_workflow_by_repo_ids( - self, repo_ids: List[str], type: RepoWorkflowType - ) -> List[RepoWorkflow]: - return ( - self._db.session.query(RepoWorkflow) - .options(defer(RepoWorkflow.meta)) - .filter( - and_( - RepoWorkflow.org_repo_id.in_(repo_ids), - RepoWorkflow.type == type, - RepoWorkflow.is_active.is_(True), - ) - ) - .all() - ) - - @rollback_on_exc - def get_repo_workflows_by_repo_id(self, repo_id: str) -> List[RepoWorkflow]: - return ( - self._db.session.query(RepoWorkflow) - .options(defer(RepoWorkflow.meta)) - .filter( - RepoWorkflow.org_repo_id == repo_id, - RepoWorkflow.is_active.is_(True), - ) - .all() - ) - - @rollback_on_exc - def get_successful_repo_workflows_runs_by_repo_ids( - self, repo_ids: List[str], interval: Interval, workflow_filter: WorkflowFilter - ) -> List[Tuple[RepoWorkflow, RepoWorkflowRuns]]: - query = ( - self._db.session.query(RepoWorkflow, RepoWorkflowRuns) - .options(defer(RepoWorkflow.meta), defer(RepoWorkflowRuns.meta)) - .join( - RepoWorkflowRuns, RepoWorkflow.id == RepoWorkflowRuns.repo_workflow_id - ) - ) - query = self._filter_active_repo_workflows(query) - query = self._filter_repo_workflows_by_repo_ids(query, repo_ids) - query = self._filter_repo_workflow_runs_in_interval(query, interval) - query = self._filter_repo_workflow_runs_status( - query, RepoWorkflowRunsStatus.SUCCESS - ) - - query = self._filter_workflows(query, workflow_filter) - - query = query.order_by(RepoWorkflowRuns.conducted_at.asc()) - - return query.all() - - @rollback_on_exc - def get_repos_workflow_runs_by_repo_ids( - self, - repo_ids: List[str], - interval: Interval, - workflow_filter: WorkflowFilter = None, - ) -> List[Tuple[RepoWorkflow, RepoWorkflowRuns]]: - query = ( - self._db.session.query(RepoWorkflow, RepoWorkflowRuns) - .options(defer(RepoWorkflow.meta), defer(RepoWorkflowRuns.meta)) - .join( - RepoWorkflowRuns, RepoWorkflow.id == RepoWorkflowRuns.repo_workflow_id - ) - ) - query = self._filter_active_repo_workflows(query) - query = self._filter_active_repo_workflows(query) - query = self._filter_repo_workflows_by_repo_ids(query, repo_ids) - query = self._filter_repo_workflow_runs_in_interval(query, interval) - - query = self._filter_workflows(query, workflow_filter) - - query = query.order_by(RepoWorkflowRuns.conducted_at.asc()) - - return query.all() - - @rollback_on_exc - def get_repo_workflow_run_by_id( - self, repo_workflow_run_id: str - ) -> Tuple[RepoWorkflow, RepoWorkflowRuns]: - return ( - self._db.session.query(RepoWorkflow, RepoWorkflowRuns) - .options(defer(RepoWorkflow.meta), defer(RepoWorkflowRuns.meta)) - .join(RepoWorkflow, RepoWorkflow.id == RepoWorkflowRuns.repo_workflow_id) - .filter(RepoWorkflowRuns.id == repo_workflow_run_id) - .one_or_none() - ) - - @rollback_on_exc - def get_previous_workflow_run( - self, workflow_run: RepoWorkflowRuns - ) -> Tuple[RepoWorkflow, RepoWorkflowRuns]: - return ( - self._db.session.query(RepoWorkflow, RepoWorkflowRuns) - .options(defer(RepoWorkflow.meta), defer(RepoWorkflowRuns.meta)) - .join(RepoWorkflow, RepoWorkflow.id == RepoWorkflowRuns.repo_workflow_id) - .filter( - RepoWorkflowRuns.repo_workflow_id == workflow_run.repo_workflow_id, - RepoWorkflowRuns.conducted_at < workflow_run.conducted_at, - RepoWorkflowRuns.head_branch == workflow_run.head_branch, - ) - .order_by(RepoWorkflowRuns.conducted_at.desc()) - .first() - ) - - @rollback_on_exc - def get_repo_workflow_runs_conducted_after_time( - self, repo_id: str, from_time: datetime = None, limit_value: int = 500 - ): - query = ( - self._db.session.query(RepoWorkflowRuns) - .options(defer(RepoWorkflowRuns.meta)) - .join(RepoWorkflow, RepoWorkflow.id == RepoWorkflowRuns.repo_workflow_id) - .filter( - RepoWorkflow.org_repo_id == repo_id, - RepoWorkflow.is_active.is_(True), - RepoWorkflowRuns.status == RepoWorkflowRunsStatus.SUCCESS, - ) - ) - - if from_time: - query = query.filter(RepoWorkflowRuns.conducted_at >= from_time) - - query = query.order_by(RepoWorkflowRuns.conducted_at) - - return query.limit(limit_value).all() - - def _filter_active_repo_workflows(self, query): - return query.filter( - RepoWorkflow.is_active.is_(True), - ) - - def _filter_repo_workflows_by_repo_ids(self, query, repo_ids: List[str]): - return query.filter(RepoWorkflow.org_repo_id.in_(repo_ids)) - - def _filter_repo_workflow_runs_in_interval(self, query, interval: Interval): - return query.filter( - RepoWorkflowRuns.conducted_at.between(interval.from_time, interval.to_time) - ) - - def _filter_repo_workflow_runs_status(self, query, status: RepoWorkflowRunsStatus): - return query.filter(RepoWorkflowRuns.status == status) - - def _filter_workflows(self, query, workflow_filter: WorkflowFilter): - if not workflow_filter: - return query - query = query.filter(*workflow_filter.filter_query) - return query diff --git a/apiserver/dora/utils/__init__.py b/apiserver/dora/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/dora/utils/cryptography.py b/apiserver/dora/utils/cryptography.py deleted file mode 100644 index 7d336b183..000000000 --- a/apiserver/dora/utils/cryptography.py +++ /dev/null @@ -1,95 +0,0 @@ -import configparser -import os -from base64 import b64encode, b64decode - -from Crypto.Cipher import PKCS1_OAEP -from Crypto.PublicKey import RSA - -service = None -CONFIG_PATH = "dora/config/config.ini" - - -class CryptoService: - def __init__(self): - self._public_key, self._public_cipher = (None, None) - self._private_key, self._private_cipher = (None, None) - - def _init_keys(self): - # Skip if already setup - if self._public_key: - return - - config = configparser.ConfigParser() - config_path = os.path.join(os.getcwd(), CONFIG_PATH) - config.read(config_path) - public_key = self._decode_key(config.get("KEYS", "SECRET_PUBLIC_KEY")) - private_key = self._decode_key(config.get("KEYS", "SECRET_PRIVATE_KEY")) - - self._public_key = RSA.importKey(public_key) if public_key else None - self._private_key = RSA.importKey(private_key) if private_key else None - - self._public_cipher = ( - PKCS1_OAEP.new(self._public_key) if self._public_key else None - ) - self._private_cipher = ( - PKCS1_OAEP.new(self._private_key) if self._private_key else None - ) - - def encrypt(self, message: str, chunk_size: int) -> [str]: - self._init_keys() - - if not message: - return message - - if not self._public_key: - raise Exception("No public key found to encrypt") - - chunks = [ - message[i : i + chunk_size] for i in range(0, len(message), chunk_size) - ] - - return [ - b64encode(self._public_cipher.encrypt(chunk.encode("utf8"))).decode("utf8") - for chunk in chunks - ] - - def decrypt(self, secret: str): - self._init_keys() - - if not secret: - return secret - - if not self._private_key: - raise Exception("No private key found to decrypt") - - secret = secret.encode("utf8") - return self._private_cipher.decrypt(b64decode(secret)).decode("utf8") - - def decrypt_chunks(self, secret_chunks: [str]): - self._init_keys() - - if not secret_chunks: - return secret_chunks - - if not self._private_key: - raise Exception("No private key found to decrypt") - - return "".join( - self._private_cipher.decrypt(b64decode(secret.encode("utf8"))).decode( - "utf8" - ) - for secret in secret_chunks - ) - - def _decode_key(self, key: str) -> str: - key = key.replace("%", "/") - key = key.encode("utf8") - return b64decode(key).decode("utf8") - - -def get_crypto_service(): - global service - if not service: - service = CryptoService() - - return service diff --git a/apiserver/dora/utils/dict.py b/apiserver/dora/utils/dict.py deleted file mode 100644 index 2bd91cf84..000000000 --- a/apiserver/dora/utils/dict.py +++ /dev/null @@ -1,32 +0,0 @@ -from typing import Dict, Any, List - - -def get_average_of_dict_values(key_to_int_map: Dict[any, int]) -> int: - """ - This method accepts a dictionary with any key type mapped to integer values and returns the average of those keys. Nulls are considered as zero. - """ - - if not key_to_int_map: - return 0 - - values = list(key_to_int_map.values()) - sum_of_value = 0 - for value in values: - - if value is None: - continue - - sum_of_value += value - - return sum_of_value // len(values) - - -def get_key_to_count_map_from_key_to_list_map( - week_to_list_map: Dict[Any, List[Any]] -) -> Dict[Any, int]: - """ - This method takes a dict of keys to list and returns a dict of keys mapped to the length of lists from the input dict. - """ - list_len_or_zero = lambda x: len(x) if type(x) in [list, set] else 0 - - return {key: list_len_or_zero(lst) for key, lst in week_to_list_map.items()} diff --git a/apiserver/dora/utils/github.py b/apiserver/dora/utils/github.py deleted file mode 100644 index ca472d70e..000000000 --- a/apiserver/dora/utils/github.py +++ /dev/null @@ -1,50 +0,0 @@ -from queue import Queue -from threading import Thread - -from github import Organization - -from dora.utils.log import LOG - - -def github_org_data_multi_thread_worker(orgs: [Organization]) -> dict: - class Worker(Thread): - def __init__(self, request_queue: Queue): - Thread.__init__(self) - self.queue = request_queue - self.results = {} - - def run(self): - while True: - if self.queue.empty(): - break - org = self.queue.get() - try: - repos = list(org.get_repos().get_page(0)[:5]) - except Exception as e: - LOG.warn(f"Error while fetching github data for {org.name}: {e}") - self.queue.task_done() - continue - self.results[org.name] = { - "repos": [repo.name for repo in repos], - } - self.queue.task_done() - - q = Queue() - num_of_workers = len(orgs) - for org in orgs: - q.put(org) - - workers = [] - for _ in range(num_of_workers): - worker = Worker(q) - worker.start() - workers.append(worker) - - for worker in workers: - worker.join() - - # Combine results from all workers - r = {} - for worker in workers: - r.update(worker.results) - return r diff --git a/apiserver/dora/utils/lock.py b/apiserver/dora/utils/lock.py deleted file mode 100644 index 527e1ae3d..000000000 --- a/apiserver/dora/utils/lock.py +++ /dev/null @@ -1,41 +0,0 @@ -from os import getenv - -from redis import Redis -from redis_lock import Lock - -REDIS_HOST = getenv("REDIS_HOST", "localhost") -REDIS_PORT = getenv("REDIS_PORT", 6379) -REDIS_DB = 0 -REDIS_PASSWORD = "" -SSL_STATUS = True if REDIS_HOST != "localhost" else False - -service = None - - -class RedisLockService: - def __init__(self, host, port, db, password, ssl): - self.host = host - self.port = port - self.db = db - self.password = password - self.ssl = ssl - self.redis = Redis( - host=self.host, - port=self.port, - db=self.db, - password=self.password, - ssl=self.ssl, - socket_connect_timeout=5, - ) - - def acquire_lock(self, key: str): - return Lock(self.redis, name=key, expire=1.5, auto_renewal=True) - - -def get_redis_lock_service(): - global service - if not service: - service = RedisLockService( - REDIS_HOST, REDIS_PORT, REDIS_DB, REDIS_PASSWORD, SSL_STATUS - ) - return service diff --git a/apiserver/dora/utils/log.py b/apiserver/dora/utils/log.py deleted file mode 100644 index c7c8882e3..000000000 --- a/apiserver/dora/utils/log.py +++ /dev/null @@ -1,17 +0,0 @@ -import logging - -LOG = logging.getLogger() - - -def custom_logging(func): - def wrapper(*args, **kwargs): - print( - f"[{func.__name__.upper()}]", args[0] - ) # Assuming the first argument is the log message - return func(*args, **kwargs) - - return wrapper - - -LOG.error = custom_logging(LOG.error) -LOG.info = custom_logging(LOG.info) diff --git a/apiserver/dora/utils/regex.py b/apiserver/dora/utils/regex.py deleted file mode 100644 index b0893ee60..000000000 --- a/apiserver/dora/utils/regex.py +++ /dev/null @@ -1,29 +0,0 @@ -import re -from typing import List -from werkzeug.exceptions import BadRequest - - -def check_regex(pattern: str): - # pattern is a string containing the regex pattern - try: - re.compile(pattern) - - except re.error: - return False - - return True - - -def check_all_regex(patterns: List[str]) -> bool: - # patterns is a list of strings containing the regex patterns - for pattern in patterns: - if not pattern or not check_regex(pattern): - return False - - return True - - -def regex_list(patterns: List[str]) -> List[str]: - if not check_all_regex(patterns): - raise BadRequest("Invalid regex pattern") - return patterns diff --git a/apiserver/dora/utils/string.py b/apiserver/dora/utils/string.py deleted file mode 100644 index a56caf067..000000000 --- a/apiserver/dora/utils/string.py +++ /dev/null @@ -1,5 +0,0 @@ -from uuid import uuid4 - - -def uuid4_str(): - return str(uuid4()) diff --git a/apiserver/dora/utils/time.py b/apiserver/dora/utils/time.py deleted file mode 100644 index 6e8b7ceb8..000000000 --- a/apiserver/dora/utils/time.py +++ /dev/null @@ -1,270 +0,0 @@ -from datetime import datetime, timedelta -from typing import Callable, List, Dict, Any, Optional -from collections import defaultdict - -import pytz - -ISO_8601_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" - - -def time_now(): - return datetime.now().astimezone(pytz.UTC) - - -class Interval: - def __init__(self, from_time: datetime, to_time: datetime): - assert ( - to_time >= from_time - ), f"from_time: {from_time.isoformat()} is greater than to_time: {to_time.isoformat()}" - self._from_time = from_time - self._to_time = to_time - - def __contains__(self, dt: datetime): - return self.from_time < dt < self.to_time - - @property - def from_time(self): - return self._from_time - - @property - def to_time(self): - return self._to_time - - @property - def duration(self) -> timedelta: - return self._to_time - self._from_time - - def overlaps(self, interval): - if interval.from_time <= self.to_time < interval.to_time: - return True - - if self.from_time <= interval.from_time < self.to_time: - return True - - return False - - def merge(self, interval): - return Interval( - min(self.from_time, interval.from_time), max(self.to_time, interval.to_time) - ) - - def merge(self, interval: []): - return Interval( - min(self.from_time, interval.from_time), max(self.to_time, interval.to_time) - ) - - def __str__(self): - return f"{self._from_time.isoformat()} -> {self._to_time.isoformat()}" - - def __repr__(self): - return str(self) - - def __eq__(self, other): - return self.from_time == other.from_time and self.to_time == other.to_time - - @staticmethod - def merge_intervals(intervals): - if not intervals or len(intervals) == 1: - return intervals - - intervals.sort(key=lambda x: (x.from_time, x.to_time)) - merged_intervals = [intervals[0]] - for interval in intervals[1:]: - if merged_intervals[-1].overlaps(interval): - merged_intervals[-1] = merged_intervals[-1].merge(interval) - else: - merged_intervals.append(interval) - - return merged_intervals - - def get_remaining_intervals(self, intervals): - if not intervals: - return [self] - - intervals = Interval.merge_intervals(intervals) - - free_intervals = [] - fro, to = self.from_time, self.to_time - for interval in intervals: - if interval.from_time > fro: - free_intervals.append(Interval(fro, interval.from_time)) - fro = interval.to_time - - if fro < to: - free_intervals.append(Interval(fro, to)) - - return free_intervals - - -def get_start_of_day(date: datetime) -> datetime: - return datetime(date.year, date.month, date.day, 0, 0, 0, tzinfo=pytz.UTC) - - -def get_end_of_day(date: datetime) -> datetime: - return datetime( - date.year, date.month, date.day, 23, 59, 59, 999999, tzinfo=pytz.UTC - ) - - -def get_given_weeks_monday(dt: datetime): - monday = dt - timedelta(days=dt.weekday()) - - monday_midnight = datetime( - monday.year, monday.month, monday.day, 0, 0, 0, tzinfo=pytz.UTC - ) - - return monday_midnight - - -def get_given_weeks_sunday(dt: datetime): - sunday = dt + timedelta(days=(6 - dt.weekday())) - sunday_midnight = datetime(sunday.year, sunday.month, sunday.day, tzinfo=pytz.UTC) - return get_end_of_day(sunday_midnight) - - -def get_time_delta_based_on_granularity(date: datetime, granularity: str) -> timedelta: - """ - Takes a date and a granularity. - Returns a timedelta based on the granularity. - Granularity options: 'daily', 'weekly', 'monthly'. - """ - if granularity == "daily": - return timedelta(days=1) - if granularity == "weekly": - return timedelta(weeks=1) - if granularity == "monthly": - some_day_in_next_month = date.replace(day=28) + timedelta(days=4) - last_day_of_month = some_day_in_next_month - timedelta( - days=some_day_in_next_month.day - ) - return last_day_of_month - date + timedelta(days=1) - raise ValueError("Invalid granularity. Choose 'daily', 'weekly', or 'monthly'.") - - -def get_expanded_interval_based_on_granularity( - interval: Interval, granularity: str -) -> Interval: - """ - Takes an interval and a granularity. - Returns an expanded interval based on the granularity. - Granularity options: 'daily', 'weekly', 'monthly'. - """ - if granularity == "daily": - return Interval( - get_start_of_day(interval.from_time), get_end_of_day(interval.to_time) - ) - if granularity == "weekly": - return Interval( - get_given_weeks_monday(interval.from_time), - get_given_weeks_sunday(interval.to_time), - ) - if granularity == "monthly": - some_day_in_next_month = interval.to_time.replace(day=28) + timedelta(days=4) - return Interval( - datetime( - interval.from_time.year, interval.from_time.month, 1, tzinfo=pytz.UTC - ), - get_end_of_day( - some_day_in_next_month - timedelta(days=some_day_in_next_month.day) - ), - ) - raise ValueError("Invalid granularity. Choose 'daily', 'weekly', or 'monthly'.") - - -def generate_expanded_buckets( - lst: List[Any], - interval: Interval, - datetime_attribute: str, - granularity: str = "weekly", -) -> Dict[datetime, List[Any]]: - """ - Takes a list of objects, time interval, a datetime_attribute string, and a granularity. - Buckets the list of objects based on the specified granularity of the datetime_attribute. - The series is expanded beyond the input interval based on the datetime_attribute. - Granularity options: 'daily', 'weekly', 'monthly'. - """ - from_time = interval.from_time - to_time = interval.to_time - - def generate_empty_buckets( - from_time: datetime, to_time: datetime, granularity: str - ) -> Dict[datetime, List[Any]]: - buckets_map: Dict[datetime, List[Any]] = defaultdict(list) - expanded_interval = get_expanded_interval_based_on_granularity( - Interval(from_time, to_time), granularity - ) - curr_date = expanded_interval.from_time - while curr_date <= expanded_interval.to_time: - delta = get_time_delta_based_on_granularity(curr_date, granularity) - buckets_map[get_start_of_day(curr_date)] = [] - curr_date += delta - - return buckets_map - - for obj in lst: - if not isinstance(getattr(obj, datetime_attribute), datetime): - raise ValueError( - f"Type of datetime_attribute {type(getattr(obj, datetime_attribute))} is not datetime" - ) - - buckets_map: Dict[datetime, List[Any]] = generate_empty_buckets( - from_time, to_time, granularity - ) - - for obj in lst: - date_value = getattr(obj, datetime_attribute) - if granularity == "daily": - bucket_key = get_start_of_day(date_value) - elif granularity == "weekly": - # Adjust the date to the start of the week (Monday) - bucket_key = get_start_of_day( - date_value - timedelta(days=date_value.weekday()) - ) - elif granularity == "monthly": - # Adjust the date to the start of the month - bucket_key = get_start_of_day(date_value.replace(day=1)) - else: - raise ValueError( - "Invalid granularity. Choose 'daily', 'weekly', or 'monthly'." - ) - - buckets_map[bucket_key].append(obj) - - return buckets_map - - -def sort_dict_by_datetime_keys(input_dict): - sorted_items = sorted(input_dict.items()) - sorted_dict = dict(sorted_items) - return sorted_dict - - -def fill_missing_week_buckets( - week_start_to_object_map: Dict[datetime, Any], - interval: Interval, - callable_class: Optional[Callable] = None, -) -> Dict[datetime, Any]: - """ - Takes a dict of week_start to object map. - Add the missing weeks with default value of the class/callable. - If no callable is passed, the missing weeks are set to None. - """ - first_monday = get_given_weeks_monday(interval.from_time) - last_sunday = get_given_weeks_sunday(interval.to_time) - - curr_day = first_monday - week_start_to_object_map_with_weeks_in_interval = {} - - while curr_day < last_sunday: - if curr_day not in week_start_to_object_map: - week_start_to_object_map_with_weeks_in_interval[curr_day] = ( - callable_class() if callable_class else None - ) - else: - week_start_to_object_map_with_weeks_in_interval[ - curr_day - ] = week_start_to_object_map[curr_day] - - curr_day = curr_day + timedelta(days=7) - - return sort_dict_by_datetime_keys(week_start_to_object_map_with_weeks_in_interval) diff --git a/apiserver/tests/__init__.py b/apiserver/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/tests/factories/__init__.py b/apiserver/tests/factories/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/tests/factories/models/__init__.py b/apiserver/tests/factories/models/__init__.py deleted file mode 100644 index 9d80b2096..000000000 --- a/apiserver/tests/factories/models/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -from .code import ( - get_repo_workflow_run, - get_deployment, - get_pull_request, - get_pull_request_commit, - get_pull_request_event, -) -from .incidents import ( - get_incident, - get_change_failure_rate_metrics, - get_org_incident_service, -) diff --git a/apiserver/tests/factories/models/code.py b/apiserver/tests/factories/models/code.py deleted file mode 100644 index 51adc2e6d..000000000 --- a/apiserver/tests/factories/models/code.py +++ /dev/null @@ -1,201 +0,0 @@ -from random import randint -from uuid import uuid4 -from dora.service.deployments.models.models import ( - Deployment, - DeploymentFrequencyMetrics, - DeploymentStatus, - DeploymentType, -) -from dora.utils.string import uuid4_str - -from dora.store.models.code import ( - PullRequestCommit, - PullRequestEvent, - PullRequestEventType, - PullRequestState, - PullRequest, - RepoWorkflowRuns, - RepoWorkflowRunsStatus, -) -from dora.utils.time import time_now - - -def get_pull_request( - id=None, - repo_id=None, - number=None, - author=None, - state=None, - title=None, - head_branch=None, - base_branch=None, - provider=None, - requested_reviews=None, - data=None, - state_changed_at=None, - created_at=None, - updated_at=None, - meta=None, - reviewers=None, - first_commit_to_open=None, - first_response_time=None, - rework_time=None, - merge_time=None, - cycle_time=None, - merge_to_deploy=None, - lead_time=None, - url=None, - merge_commit_sha=None, -): - return PullRequest( - id=id or uuid4(), - repo_id=repo_id or uuid4(), - number=number or randint(10, 100), - author=author or "randomuser", - title=title or "title", - state=state or PullRequestState.OPEN, - head_branch=head_branch or "feature", - base_branch=base_branch or "main", - provider=provider or "github", - requested_reviews=requested_reviews or [], - data=data or {}, - state_changed_at=state_changed_at or time_now(), - created_at=created_at or time_now(), - updated_at=updated_at or time_now(), - first_commit_to_open=first_commit_to_open, - first_response_time=first_response_time, - rework_time=rework_time, - merge_time=merge_time, - cycle_time=cycle_time, - merge_to_deploy=merge_to_deploy, - lead_time=lead_time, - reviewers=reviewers - if reviewers is not None - else ["randomuser1", "randomuser2"], - meta=meta or {}, - url=url, - merge_commit_sha=merge_commit_sha, - ) - - -def get_pull_request_event( - id=None, - pull_request_id=None, - type=None, - reviewer=None, - state=None, - created_at=None, - idempotency_key=None, - org_repo_id=None, - data=None, -): - return PullRequestEvent( - id=id or uuid4(), - pull_request_id=pull_request_id or uuid4(), - type=type or PullRequestEventType.REVIEW.value, - data={ - "user": {"login": reviewer or "User"}, - "state": state or "APPROVED", - "author_association": "NONE", - } - if not data - else data, - created_at=created_at or time_now(), - idempotency_key=idempotency_key or str(randint(10, 100)), - org_repo_id=org_repo_id or uuid4(), - actor_username=reviewer or "randomuser", - ) - - -def get_pull_request_commit( - hash=None, - pr_id=None, - message=None, - url=None, - data=None, - author=None, - created_at=None, - org_repo_id=None, -): - return PullRequestCommit( - hash=hash or uuid4(), - pull_request_id=pr_id or uuid4(), - message=message or "message", - url=url or "https://abc.com", - data=data or dict(), - author=author or "randomuser", - created_at=created_at or time_now(), - org_repo_id=org_repo_id or uuid4(), - ) - - -def get_repo_workflow_run( - id=None, - repo_workflow_id=None, - provider_workflow_run_id=None, - event_actor=None, - head_branch=None, - status=None, - conducted_at=None, - created_at=None, - updated_at=None, - meta=None, - duration=None, - html_url=None, -): - return RepoWorkflowRuns( - id=id or uuid4(), - repo_workflow_id=repo_workflow_id or uuid4(), - provider_workflow_run_id=provider_workflow_run_id or "1234567", - event_actor=event_actor or "samad-yar-khan", - head_branch=head_branch or "master", - status=status or RepoWorkflowRunsStatus.SUCCESS, - conducted_at=conducted_at or time_now(), - created_at=created_at or time_now(), - updated_at=updated_at or time_now(), - duration=duration, - meta=meta, - html_url=html_url, - ) - - -def get_deployment( - repo_id=None, - entity_id=None, - actor=None, - head_branch=None, - status=None, - conducted_at=None, - meta=None, - duration=None, - html_url=None, - provider=None, -): - return Deployment( - deployment_type=DeploymentType.WORKFLOW, - repo_id=repo_id or "1234567", - entity_id=entity_id or uuid4_str(), - provider=provider or "github", - actor=actor or "samad-yar-khan", - head_branch=head_branch or "master", - conducted_at=conducted_at or time_now(), - duration=duration, - status=status or DeploymentStatus.SUCCESS, - html_url=html_url or "", - meta=meta or {}, - ) - - -def get_deployment_frequency_metrics( - total_deployments=0, - daily_deployment_frequency=0, - avg_weekly_deployment_frequency=0, - avg_monthly_deployment_frequency=0, -) -> DeploymentFrequencyMetrics: - - return DeploymentFrequencyMetrics( - total_deployments=total_deployments or 0, - daily_deployment_frequency=daily_deployment_frequency or 0, - avg_weekly_deployment_frequency=avg_weekly_deployment_frequency or 0, - avg_monthly_deployment_frequency=avg_monthly_deployment_frequency or 0, - ) diff --git a/apiserver/tests/factories/models/exapi/__init__.py b/apiserver/tests/factories/models/exapi/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/tests/factories/models/exapi/github.py b/apiserver/tests/factories/models/exapi/github.py deleted file mode 100644 index 8dd11a868..000000000 --- a/apiserver/tests/factories/models/exapi/github.py +++ /dev/null @@ -1,159 +0,0 @@ -from collections import namedtuple -from dataclasses import dataclass -from datetime import datetime -from typing import Dict - -from dora.utils.time import time_now - - -def get_github_commit_dict( - sha: str = "123456789098765", - author_login: str = "author_abc", - url: str = "https://github.com/123456789098765", - message: str = "[abc 315] avoid mapping edit state", - created_at: str = "2022-06-29T10:53:15Z", -) -> Dict: - return { - "sha": sha, - "commit": { - "committer": {"name": "abc", "email": "abc@midd.com", "date": created_at}, - "message": message, - }, - "author": { - "login": author_login, - "id": 95607047, - "node_id": "abc", - "avatar_url": "", - }, - "html_url": url, - } - - -@dataclass -class GithubPullRequestReview: - id: str - submitted_at: datetime - user_login: str - - @property - def raw_data(self): - return { - "id": self.id, - "submitted_at": self.submitted_at, - "user": { - "login": self.user_login, - }, - } - - -def get_github_pull_request_review( - review_id: str = "123456", - submitted_at: datetime = time_now(), - user_login: str = "abc", -) -> GithubPullRequestReview: - - return GithubPullRequestReview(review_id, submitted_at, user_login) - - -Branch = namedtuple("Branch", ["ref"]) -User = namedtuple("User", ["login"]) - - -@dataclass -class GithubPullRequest: - number: int - merged_at: datetime - closed_at: datetime - title: str - html_url: str - created_at: datetime - updated_at: datetime - base: Branch - head: Branch - user: User - commits: int - additions: int - deletions: int - changed_files: int - merge_commit_sha: str - - @property - def raw_data(self): - return { - "number": self.number, - "merged_at": self.merged_at, - "closed_at": self.closed_at, - "title": self.title, - "html_url": self.html_url, - "created_at": self.created_at, - "updated_at": self.updated_at, - "base": {"ref": self.base.ref}, - "head": {"ref": self.head.ref}, - "user": {"login": self.user.login}, - "commits": self.commits, - "additions": self.additions, - "deletions": self.deletions, - "changed_files": self.changed_files, - "requested_reviewers": [], - "merge_commit_sha": self.merge_commit_sha, - } - - -def get_github_pull_request( - number: int = 1, - merged_at: datetime = None, - closed_at: datetime = None, - title: str = "random_title", - html_url: str = None, - created_at: datetime = time_now(), - updated_at: datetime = time_now(), - base_ref: str = "main", - head_ref: str = "feature", - user_login: str = "abc", - commits: int = 1, - additions: int = 1, - deletions: int = 1, - changed_files: int = 1, - merge_commit_sha: str = "123456", -) -> GithubPullRequest: - return GithubPullRequest( - number, - merged_at, - closed_at, - title, - html_url, - created_at, - updated_at, - Branch(base_ref), - Branch(head_ref), - User(user_login), - commits, - additions, - deletions, - changed_files, - merge_commit_sha, - ) - - -def get_github_workflow_run_dict( - run_id: str = "123456", - actor_login: str = "abc", - head_branch: str = "feature", - status: str = "completed", - conclusion: str = "success", - run_started_at: str = "2022-06-29T10:53:15Z", - created_at: str = "2022-06-29T10:53:15Z", - updated_at: str = "2022-06-29T10:53:15Z", - html_url: str = "", -) -> Dict: - return { - "id": run_id, - "actor": {"login": actor_login}, - "head_branch": head_branch, - "status": status, - "conclusion": conclusion, - "run_started_at": run_started_at, - "created_at": created_at, - "updated_at": updated_at, - "html_url": html_url, - } diff --git a/apiserver/tests/factories/models/incidents.py b/apiserver/tests/factories/models/incidents.py deleted file mode 100644 index bf0c3af42..000000000 --- a/apiserver/tests/factories/models/incidents.py +++ /dev/null @@ -1,93 +0,0 @@ -from datetime import datetime -from typing import List, Set - -from voluptuous import default_factory -from dora.service.deployments.models.models import Deployment -from dora.service.incidents.models.mean_time_to_recovery import ChangeFailureRateMetrics - -from dora.store.models.incidents import IncidentType, OrgIncidentService -from dora.store.models.incidents.incidents import ( - Incident, - IncidentOrgIncidentServiceMap, -) -from dora.utils.string import uuid4_str -from dora.utils.time import time_now - - -def get_incident( - id: str = uuid4_str(), - provider: str = "provider", - key: str = "key", - title: str = "title", - status: str = "status", - incident_number: int = 0, - incident_type: IncidentType = IncidentType("INCIDENT"), - creation_date: datetime = time_now(), - created_at: datetime = time_now(), - updated_at: datetime = time_now(), - resolved_date: datetime = time_now(), - acknowledged_date: datetime = time_now(), - assigned_to: str = "assigned_to", - assignees: List[str] = default_factory(list), - meta: dict = default_factory(dict), -) -> Incident: - return Incident( - id=id, - provider=provider, - key=key, - title=title, - status=status, - incident_number=incident_number, - incident_type=incident_type, - created_at=created_at, - updated_at=updated_at, - creation_date=creation_date, - resolved_date=resolved_date, - assigned_to=assigned_to, - assignees=assignees, - acknowledged_date=acknowledged_date, - meta=meta, - ) - - -def get_org_incident_service( - service_id: str, - org_id: str = uuid4_str(), - name: str = "Service", - provider: str = "PagerDuty", - key: str = "service_key", - auto_resolve_timeout: int = 0, - acknowledgement_timeout: int = 0, - created_by: str = "user", - provider_team_keys=default_factory(list), - status: str = "active", - meta: dict = default_factory(dict), -): - return OrgIncidentService( - id=service_id if service_id else uuid4_str(), - org_id=org_id if org_id else uuid4_str(), - name=name, - provider=provider, - key=key, - auto_resolve_timeout=auto_resolve_timeout, - acknowledgement_timeout=acknowledgement_timeout, - created_by=created_by, - provider_team_keys=provider_team_keys, - status=status, - meta=meta, - created_at=time_now(), - updated_at=time_now(), - ) - - -def get_incident_org_incident_map( - incident_id: str = uuid4_str(), service_id: str = uuid4_str() -): - return IncidentOrgIncidentServiceMap(incident_id=incident_id, service_id=service_id) - - -def get_change_failure_rate_metrics( - failed_deployments: Set[Deployment] = None, - total_deployments: Set[Deployment] = None, -): - return ChangeFailureRateMetrics(failed_deployments, total_deployments) diff --git a/apiserver/tests/service/Incidents/sync/__init__.py b/apiserver/tests/service/Incidents/sync/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/tests/service/Incidents/sync/test_etl_git_incidents_handler.py b/apiserver/tests/service/Incidents/sync/test_etl_git_incidents_handler.py deleted file mode 100644 index a8aab0252..000000000 --- a/apiserver/tests/service/Incidents/sync/test_etl_git_incidents_handler.py +++ /dev/null @@ -1,141 +0,0 @@ -from dora.exapi.models.git_incidents import RevertPRMap -from dora.service.incidents.sync.etl_git_incidents_handler import GitIncidentsETLHandler -from dora.store.models.incidents import IncidentType, IncidentStatus -from dora.utils.string import uuid4_str -from dora.utils.time import time_now -from tests.factories.models import get_incident -from tests.factories.models.code import get_pull_request -from tests.factories.models.incidents import ( - get_org_incident_service, - get_incident_org_incident_map, -) -from tests.utilities import compare_objects_as_dicts - -org_id = uuid4_str() -repo_id = uuid4_str() -provider = "github" - -original_pr = get_pull_request( - id=uuid4_str(), - title="Testing PR", - repo_id=repo_id, - head_branch="feature", - base_branch="main", - provider=provider, -) - -revert_pr = get_pull_request( - id=uuid4_str(), - title='Revert "Testing PR"', - repo_id=repo_id, - head_branch="revert-feature", - base_branch="main", - provider=provider, -) - -expected_git_incident = get_incident( - id=uuid4_str(), - provider=provider, - key=str(original_pr.id), - title=original_pr.title, - incident_number=int(original_pr.number), - status=IncidentStatus.RESOLVED.value, - creation_date=original_pr.state_changed_at, - acknowledged_date=revert_pr.created_at, - resolved_date=revert_pr.state_changed_at, - assigned_to=revert_pr.author, - assignees=[revert_pr.author], - meta={ - "revert_pr": GitIncidentsETLHandler._adapt_pr_to_json(revert_pr), - "original_pr": GitIncidentsETLHandler._adapt_pr_to_json(original_pr), - "created_at": revert_pr.created_at.isoformat(), - "updated_at": revert_pr.updated_at.isoformat(), - }, - created_at=time_now(), - updated_at=time_now(), - incident_type=IncidentType.REVERT_PR, -) - - -def test_process_revert_pr_incident_given_existing_incident_map_returns_same_incident(): - class FakeIncidentsRepoService: - def get_incident_by_key_type_and_provider( - self, - *args, - **kwargs, - ): - return expected_git_incident - - git_incident_service = GitIncidentsETLHandler( - org_id, None, FakeIncidentsRepoService() - ) - - org_incident_service = get_org_incident_service( - provider="github", service_id=repo_id - ) - - revert_pr_map = RevertPRMap( - original_pr=original_pr, - revert_pr=revert_pr, - created_at=time_now(), - updated_at=time_now(), - ) - - incident, incident_service_map = git_incident_service._process_revert_pr_incident( - org_incident_service, revert_pr_map - ) - - expected_incident_org_incident_service_map = get_incident_org_incident_map( - expected_git_incident.id, service_id=repo_id - ) - - assert compare_objects_as_dicts( - expected_git_incident, incident, ["created_at", "updated_at"] - ) - - assert compare_objects_as_dicts( - expected_incident_org_incident_service_map, incident_service_map - ) - - -def test_process_revert_pr_incident_given_no_existing_incident_map_returns_new_incident(): - class FakeIncidentsRepoService: - def get_incident_by_key_type_and_provider( - self, - *args, - **kwargs, - ): - return None - - git_incident_service = GitIncidentsETLHandler( - org_id, None, FakeIncidentsRepoService() - ) - - org_incident_service = get_org_incident_service( - provider="github", service_id=repo_id - ) - - revert_pr_map = RevertPRMap( - original_pr=original_pr, - revert_pr=revert_pr, - created_at=time_now(), - updated_at=time_now(), - ) - - incident, incident_service_map = git_incident_service._process_revert_pr_incident( - org_incident_service, revert_pr_map - ) - - assert compare_objects_as_dicts( - expected_git_incident, incident, ["id", "created_at", "updated_at"] - ) - - expected_incident_org_incident_service_map = get_incident_org_incident_map( - uuid4_str(), service_id=repo_id - ) - - assert compare_objects_as_dicts( - expected_incident_org_incident_service_map, - incident_service_map, - ["incident_id"], - ) diff --git a/apiserver/tests/service/Incidents/test_change_failure_rate.py b/apiserver/tests/service/Incidents/test_change_failure_rate.py deleted file mode 100644 index 2f5e80945..000000000 --- a/apiserver/tests/service/Incidents/test_change_failure_rate.py +++ /dev/null @@ -1,352 +0,0 @@ -import pytz -from datetime import datetime -from datetime import timedelta -from tests.factories.models.incidents import get_change_failure_rate_metrics -from dora.service.incidents.incidents import get_incident_service -from dora.utils.time import Interval, time_now - -from tests.factories.models import get_incident, get_deployment - - -# No incidents, no deployments -def test_get_change_failure_rate_for_no_incidents_no_deployments(): - incident_service = get_incident_service() - incidents = [] - deployments = [] - change_failure_rate = incident_service.get_change_failure_rate_metrics( - deployments, - incidents, - ) - assert change_failure_rate == get_change_failure_rate_metrics([], []) - assert change_failure_rate.change_failure_rate == 0 - - -# No incidents, some deployments -def test_get_change_failure_rate_for_no_incidents_and_some_deployments(): - incident_service = get_incident_service() - incidents = [] - - deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=2)) - deployment_2 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) - - deployments = [ - deployment_1, - deployment_2, - ] - change_failure_rate = incident_service.get_change_failure_rate_metrics( - deployments, - incidents, - ) - assert change_failure_rate == get_change_failure_rate_metrics( - set(), set([deployment_2, deployment_1]) - ) - assert change_failure_rate.change_failure_rate == 0 - - -# Some incidents, no deployments -def test_get_deployment_incidents_count_map_returns_empty_dict_when_given_some_incidents_no_deployments(): - incident_service = get_incident_service() - incidents = [get_incident(creation_date=time_now() - timedelta(days=3))] - deployments = [] - change_failure_rate = incident_service.get_change_failure_rate_metrics( - deployments, - incidents, - ) - assert change_failure_rate == get_change_failure_rate_metrics(set(), set()) - assert change_failure_rate.change_failure_rate == 0 - - -# One incident between two deployments -def test_get_change_failure_rate_for_one_incidents_bw_two_deployments(): - incident_service = get_incident_service() - incidents = [get_incident(creation_date=time_now() - timedelta(days=1))] - - deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=2)) - deployment_2 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) - - deployments = [ - deployment_1, - deployment_2, - ] - - change_failure_rate = incident_service.get_change_failure_rate_metrics( - deployments, - incidents, - ) - assert change_failure_rate == get_change_failure_rate_metrics( - set([deployment_1]), set([deployment_2, deployment_1]) - ) - assert change_failure_rate.change_failure_rate == 50 - - -# One incident before two deployments -def test_get_change_failure_rate_for_one_incidents_bef_two_deployments(): - incident_service = get_incident_service() - incidents = [get_incident(creation_date=time_now() - timedelta(days=3))] - - deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=2)) - deployment_2 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) - - deployments = [ - deployment_1, - deployment_2, - ] - - change_failure_rate = incident_service.get_change_failure_rate_metrics( - deployments, - incidents, - ) - assert change_failure_rate == get_change_failure_rate_metrics( - set([]), set([deployment_2, deployment_1]) - ) - assert change_failure_rate.change_failure_rate == 0 - - -# One incident after two deployments -def test_get_change_failure_rate_for_one_incidents_after_two_deployments(): - incident_service = get_incident_service() - incidents = [get_incident(creation_date=time_now() - timedelta(hours=1))] - deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=2)) - deployment_2 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) - - deployments = [ - deployment_1, - deployment_2, - ] - - change_failure_rate = incident_service.get_change_failure_rate_metrics( - deployments, - incidents, - ) - assert change_failure_rate == get_change_failure_rate_metrics( - set([deployment_2]), set([deployment_2, deployment_1]) - ) - assert change_failure_rate.change_failure_rate == 50 - - -# Multiple incidents and deployments -def test_get_change_failure_rate_for_multi_incidents_multi_deployments(): - - incident_service = get_incident_service() - - incident_0 = get_incident(creation_date=time_now() - timedelta(days=10)) - - deployment_1 = get_deployment(conducted_at=time_now() - timedelta(days=7)) - - deployment_2 = get_deployment(conducted_at=time_now() - timedelta(days=6)) - incident_1 = get_incident(creation_date=time_now() - timedelta(days=5)) - - deployment_3 = get_deployment(conducted_at=time_now() - timedelta(days=4)) - incident_2 = get_incident(creation_date=time_now() - timedelta(days=3)) - - deployment_4 = get_deployment(conducted_at=time_now() - timedelta(days=2)) - incident_3 = get_incident(creation_date=time_now() - timedelta(hours=20)) - - deployment_5 = get_deployment(conducted_at=time_now() - timedelta(hours=6)) - incident_4 = get_incident(creation_date=time_now() - timedelta(hours=4)) - incident_5 = get_incident(creation_date=time_now() - timedelta(hours=2)) - incident_6 = get_incident(creation_date=time_now() - timedelta(hours=1)) - - deployment_6 = get_deployment(conducted_at=time_now() - timedelta(minutes=30)) - - incidents = [ - incident_0, - incident_1, - incident_2, - incident_3, - incident_4, - incident_5, - incident_6, - ] - - deployments = [ - deployment_1, - deployment_2, - deployment_3, - deployment_4, - deployment_5, - deployment_6, - ] - - change_failure_rate = incident_service.get_change_failure_rate_metrics( - deployments, - incidents, - ) - - assert change_failure_rate == get_change_failure_rate_metrics( - set([deployment_2, deployment_3, deployment_4, deployment_5]), - set( - [ - deployment_1, - deployment_2, - deployment_3, - deployment_4, - deployment_5, - deployment_6, - ] - ), - ) - assert change_failure_rate.change_failure_rate == (4 / 6 * 100) - - -# No Incidents and Deployments -def test_get_weekly_change_failure_rate_for_no_incidents_no_deployments(): - - first_week_2024 = datetime(2024, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) - second_week_2024 = datetime(2024, 1, 8, 0, 0, 0, tzinfo=pytz.UTC) - third_week_2024 = datetime(2024, 1, 15, 0, 0, 0, tzinfo=pytz.UTC) - - from_time = first_week_2024 + timedelta(days=1) - to_time = third_week_2024 + timedelta(days=2) - - incidents = [] - deployments = [] - - incident_service = get_incident_service() - weekly_change_failure_rate = incident_service.get_weekly_change_failure_rate( - Interval(from_time, to_time), - deployments, - incidents, - ) - assert weekly_change_failure_rate == { - first_week_2024: get_change_failure_rate_metrics([], []), - second_week_2024: get_change_failure_rate_metrics([], []), - third_week_2024: get_change_failure_rate_metrics([], []), - } - - -# No Incidents and Deployments -def test_get_weekly_change_failure_rate_for_no_incidents_and_some_deployments(): - - first_week_2024 = datetime(2024, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) - second_week_2024 = datetime(2024, 1, 8, 0, 0, 0, tzinfo=pytz.UTC) - third_week_2024 = datetime(2024, 1, 15, 0, 0, 0, tzinfo=pytz.UTC) - - from_time = first_week_2024 + timedelta(days=1) - to_time = third_week_2024 + timedelta(days=2) - - deployment_1 = get_deployment(conducted_at=from_time + timedelta(days=2)) - deployment_2 = get_deployment(conducted_at=second_week_2024 + timedelta(days=2)) - deployment_3 = get_deployment(conducted_at=to_time - timedelta(hours=2)) - - deployments = [deployment_1, deployment_2, deployment_3] - - incidents = [] - - incident_service = get_incident_service() - weekly_change_failure_rate = incident_service.get_weekly_change_failure_rate( - Interval(from_time, to_time), - deployments, - incidents, - ) - - assert weekly_change_failure_rate == { - first_week_2024: get_change_failure_rate_metrics([], set([deployment_1])), - second_week_2024: get_change_failure_rate_metrics([], set([deployment_2])), - third_week_2024: get_change_failure_rate_metrics([], set([deployment_3])), - } - - -# No Incidents and Deployments -def test_get_weekly_change_failure_rate_for_incidents_and_no_deployments(): - - first_week_2024 = datetime(2024, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) - second_week_2024 = datetime(2024, 1, 8, 0, 0, 0, tzinfo=pytz.UTC) - third_week_2024 = datetime(2024, 1, 15, 0, 0, 0, tzinfo=pytz.UTC) - - from_time = first_week_2024 + timedelta(days=1) - to_time = third_week_2024 + timedelta(days=2) - - incident_1 = get_incident(creation_date=to_time - timedelta(days=14)) - incident_2 = get_incident(creation_date=to_time - timedelta(days=10)) - incident_3 = get_incident(creation_date=to_time - timedelta(days=7)) - incident_4 = get_incident(creation_date=to_time - timedelta(days=3)) - incident_5 = get_incident(creation_date=to_time - timedelta(hours=2)) - incident_6 = get_incident(creation_date=to_time - timedelta(hours=1)) - - incidents = [incident_1, incident_2, incident_3, incident_4, incident_5, incident_6] - deployments = [] - - incident_service = get_incident_service() - weekly_change_failure_rate = incident_service.get_weekly_change_failure_rate( - Interval(from_time, to_time), - deployments, - incidents, - ) - - assert weekly_change_failure_rate == { - first_week_2024: get_change_failure_rate_metrics([], []), - second_week_2024: get_change_failure_rate_metrics([], []), - third_week_2024: get_change_failure_rate_metrics([], []), - } - - -def test_get_weekly_change_failure_rate_for_incidents_and_deployments(): - - first_week_2024 = datetime(2024, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) - second_week_2024 = datetime(2024, 1, 8, 0, 0, 0, tzinfo=pytz.UTC) - third_week_2024 = datetime(2024, 1, 15, 0, 0, 0, tzinfo=pytz.UTC) - fourth_week_2024 = datetime(2024, 1, 22, 0, 0, 0, tzinfo=pytz.UTC) - - from_time = first_week_2024 + timedelta(days=1) - to_time = fourth_week_2024 + timedelta(days=2) - - # Week 1 - incident_0 = get_incident(creation_date=from_time + timedelta(hours=10)) - - deployment_1 = get_deployment(conducted_at=from_time + timedelta(hours=12)) - - deployment_2 = get_deployment(conducted_at=from_time + timedelta(hours=24)) - incident_1 = get_incident(creation_date=from_time + timedelta(hours=28)) - incident_2 = get_incident(creation_date=from_time + timedelta(hours=29)) - - # Week 3 - deployment_3 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=4)) - incident_3 = get_incident(creation_date=fourth_week_2024 - timedelta(days=3)) - incident_4 = get_incident(creation_date=fourth_week_2024 - timedelta(days=3)) - - deployment_4 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=2)) - - deployment_5 = get_deployment(conducted_at=fourth_week_2024 - timedelta(hours=6)) - incident_5 = get_incident(creation_date=fourth_week_2024 - timedelta(hours=3)) - incident_6 = get_incident(creation_date=fourth_week_2024 - timedelta(hours=2)) - - deployment_6 = get_deployment(conducted_at=fourth_week_2024 - timedelta(minutes=30)) - - incidents = [ - incident_0, - incident_1, - incident_2, - incident_3, - incident_4, - incident_5, - incident_6, - ] - - deployments = [ - deployment_1, - deployment_2, - deployment_3, - deployment_4, - deployment_5, - deployment_6, - ] - - incident_service = get_incident_service() - weekly_change_failure_rate = incident_service.get_weekly_change_failure_rate( - Interval(from_time, to_time), - deployments, - incidents, - ) - - assert weekly_change_failure_rate == { - first_week_2024: get_change_failure_rate_metrics( - set([deployment_2]), set([deployment_1, deployment_2]) - ), - second_week_2024: get_change_failure_rate_metrics([], []), - third_week_2024: get_change_failure_rate_metrics( - set([deployment_3, deployment_5]), - set([deployment_3, deployment_4, deployment_5, deployment_6]), - ), - fourth_week_2024: get_change_failure_rate_metrics([], []), - } diff --git a/apiserver/tests/service/Incidents/test_deployment_incident_mapper.py b/apiserver/tests/service/Incidents/test_deployment_incident_mapper.py deleted file mode 100644 index c4de193c1..000000000 --- a/apiserver/tests/service/Incidents/test_deployment_incident_mapper.py +++ /dev/null @@ -1,119 +0,0 @@ -from datetime import timedelta -from dora.service.incidents.incidents import get_incident_service -from dora.utils.time import time_now - -from tests.factories.models import get_incident, get_deployment - - -# No incidents, no deployments -def test_get_deployment_incidents_count_map_returns_empty_dict_when_given_no_incidents_no_deployments(): - incident_service = get_incident_service() - incidents = [] - deployments = [] - deployment_incidents_count_map = incident_service.get_deployment_incidents_map( - deployments, - incidents, - ) - assert deployment_incidents_count_map == {} - - -# No incidents, some deployments -def test_get_deployment_incidents_count_map_returns_deployment_incident_count_map_when_given_no_incidents_some_deployments(): - incident_service = get_incident_service() - incidents = [] - deployments = [ - get_deployment(conducted_at=time_now() - timedelta(days=2)), - get_deployment(conducted_at=time_now() - timedelta(hours=6)), - ] - deployment_incidents_count_map = incident_service.get_deployment_incidents_map( - deployments, - incidents, - ) - assert deployment_incidents_count_map == {deployments[0]: [], deployments[1]: []} - - -# Some incidents, no deployments -def test_get_deployment_incidents_count_map_returns_empty_dict_when_given_some_incidents_no_deployments(): - incident_service = get_incident_service() - incidents = [get_incident(creation_date=time_now() - timedelta(days=3))] - deployments = [] - deployment_incidents_count_map = incident_service.get_deployment_incidents_map( - deployments, incidents - ) - assert deployment_incidents_count_map == {} - - -# One incident between two deployments -def test_get_deployment_incidents_count_map_returns_deployment_incident_count_map_when_given_one_incidents_bw_two_deployments(): - incident_service = get_incident_service() - incidents = [get_incident(creation_date=time_now() - timedelta(days=1))] - deployments = [ - get_deployment(conducted_at=time_now() - timedelta(days=2)), - get_deployment(conducted_at=time_now() - timedelta(hours=6)), - ] - deployment_incidents_count_map = incident_service.get_deployment_incidents_map( - deployments, incidents - ) - assert deployment_incidents_count_map == { - deployments[0]: [incidents[0]], - deployments[1]: [], - } - - -# One incident before two deployments -def test_get_deployment_incidents_count_map_returns_deployment_incident_count_map_when_given_one_incidents_bef_two_deployments(): - incident_service = get_incident_service() - incidents = [get_incident(creation_date=time_now() - timedelta(days=3))] - deployments = [ - get_deployment(conducted_at=time_now() - timedelta(days=2)), - get_deployment(conducted_at=time_now() - timedelta(hours=6)), - ] - deployment_incidents_count_map = incident_service.get_deployment_incidents_map( - deployments, incidents - ) - assert deployment_incidents_count_map == {deployments[0]: [], deployments[1]: []} - - -# One incident after two deployments -def test_get_deployment_incidents_count_map_returns_deployment_incident_count_map_when_given_one_incidents_after_two_deployments(): - incident_service = get_incident_service() - incidents = [get_incident(creation_date=time_now() - timedelta(hours=1))] - deployments = [ - get_deployment(conducted_at=time_now() - timedelta(days=2)), - get_deployment(conducted_at=time_now() - timedelta(hours=6)), - ] - deployment_incidents_count_map = incident_service.get_deployment_incidents_map( - deployments, incidents - ) - assert deployment_incidents_count_map == { - deployments[0]: [], - deployments[1]: [incidents[0]], - } - - -# Multiple incidents and deployments -def test_get_deployment_incidents_count_map_returns_deployment_incident_count_map_when_given_multi_incidents_multi_deployments(): - incident_service = get_incident_service() - incidents = [ - get_incident(creation_date=time_now() - timedelta(days=5)), - get_incident(creation_date=time_now() - timedelta(days=3)), - get_incident(creation_date=time_now() - timedelta(hours=20)), - get_incident(creation_date=time_now() - timedelta(hours=1)), - ] - deployments = [ - get_deployment(conducted_at=time_now() - timedelta(days=7)), - get_deployment(conducted_at=time_now() - timedelta(days=6)), - get_deployment(conducted_at=time_now() - timedelta(days=4)), - get_deployment(conducted_at=time_now() - timedelta(days=2)), - get_deployment(conducted_at=time_now() - timedelta(hours=6)), - ] - deployment_incidents_count_map = incident_service.get_deployment_incidents_map( - deployments, incidents - ) - assert deployment_incidents_count_map == { - deployments[0]: [], - deployments[1]: [incidents[0]], - deployments[2]: [incidents[1]], - deployments[3]: [incidents[2]], - deployments[4]: [incidents[3]], - } diff --git a/apiserver/tests/service/__init__.py b/apiserver/tests/service/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/tests/service/code/__init__.py b/apiserver/tests/service/code/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/tests/service/code/sync/__init__.py b/apiserver/tests/service/code/sync/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/tests/service/code/sync/test_etl_code_analytics.py b/apiserver/tests/service/code/sync/test_etl_code_analytics.py deleted file mode 100644 index 740c0d90c..000000000 --- a/apiserver/tests/service/code/sync/test_etl_code_analytics.py +++ /dev/null @@ -1,505 +0,0 @@ -from datetime import timedelta - -from dora.service.code.sync.etl_code_analytics import CodeETLAnalyticsService -from dora.store.models.code import PullRequestState, PullRequestEventState -from dora.utils.time import time_now -from tests.factories.models.code import ( - get_pull_request, - get_pull_request_event, - get_pull_request_commit, -) - - -def test_pr_performance_returns_first_review_tat_for_first_review(): - pr_service = CodeETLAnalyticsService() - t1 = time_now() - t2 = t1 + timedelta(hours=1) - pr = get_pull_request(created_at=t1, updated_at=t1) - pr_event = get_pull_request_event(pull_request_id=pr.id, created_at=t2) - performance = pr_service.get_pr_performance(pr, [pr_event]) - assert performance.first_review_time == 3600 - - -def test_pr_performance_returns_minus1_first_review_tat_for_no_reviews(): - pr_service = CodeETLAnalyticsService() - pr = get_pull_request() - performance = pr_service.get_pr_performance(pr, []) - assert performance.first_review_time == -1 - - -def test_pr_performance_returns_minus1_first_approved_review_tat_for_no_approved_review(): - pr_service = CodeETLAnalyticsService() - t1 = time_now() - t2 = t1 + timedelta(hours=1) - pr = get_pull_request(created_at=t1, updated_at=t1) - pr_event_1 = get_pull_request_event( - pull_request_id=pr.id, state="REJECTED", created_at=t2 - ) - performance = pr_service.get_pr_performance(pr, [pr_event_1]) - assert performance.merge_time == -1 - - -def test_pr_performance_returns_merge_time_minus1_for_merged_pr_without_review(): - pr_service = CodeETLAnalyticsService() - t1 = time_now() - t2 = time_now() + timedelta(minutes=30) - pr = get_pull_request( - state=PullRequestState.MERGED, state_changed_at=t2, created_at=t1, updated_at=t2 - ) - performance = pr_service.get_pr_performance(pr, []) - assert performance.merge_time == -1 - - -def test_pr_performance_returns_blocking_reviews(): - pr_service = CodeETLAnalyticsService() - t1 = time_now() - t2 = time_now() + timedelta(minutes=30) - pr = get_pull_request( - state=PullRequestState.MERGED, - requested_reviews=["abc", "bcd"], - state_changed_at=t2, - created_at=t1, - updated_at=t2, - ) - performance = pr_service.get_pr_performance(pr, []) - assert performance.blocking_reviews == 0 - - -def test_pr_performance_returns_rework_time(): - pr_service = CodeETLAnalyticsService() - t1 = time_now() - t2 = t1 + timedelta(hours=1) - t3 = t2 + timedelta(hours=1) - t4 = t3 + timedelta(hours=1) - pr = get_pull_request( - state=PullRequestState.MERGED, state_changed_at=t4, created_at=t1, updated_at=t1 - ) - changes_requested_1 = get_pull_request_event( - pull_request_id=pr.id, - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t2, - ) - comment2 = get_pull_request_event( - pull_request_id=pr.id, - state=PullRequestEventState.COMMENTED.value, - created_at=t3, - ) - approval = get_pull_request_event( - pull_request_id=pr.id, state=PullRequestEventState.APPROVED.value, created_at=t4 - ) - performance = pr_service.get_pr_performance( - pr, [changes_requested_1, comment2, approval] - ) - - assert performance.rework_time == (t4 - t2).total_seconds() - - -def test_pr_performance_returns_rework_time_0_for_approved_prs(): - pr_service = CodeETLAnalyticsService() - t1 = time_now() - t2 = t1 + timedelta(hours=1) - pr = get_pull_request( - state=PullRequestState.MERGED, state_changed_at=t2, created_at=t1, updated_at=t1 - ) - approval = get_pull_request_event( - pull_request_id=pr.id, state=PullRequestEventState.APPROVED.value, created_at=t2 - ) - performance = pr_service.get_pr_performance(pr, [approval]) - - assert performance.rework_time == 0 - - -def test_pr_performance_returns_rework_time_as_per_first_approved_prs(): - pr_service = CodeETLAnalyticsService() - t1 = time_now() - t2 = t1 + timedelta(hours=1) - t3 = t2 + timedelta(hours=1) - t4 = t3 + timedelta(hours=1) - pr = get_pull_request( - state=PullRequestState.MERGED, state_changed_at=t4, created_at=t1, updated_at=t1 - ) - changes_requested_1 = get_pull_request_event( - pull_request_id=pr.id, - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t2, - ) - approval = get_pull_request_event( - pull_request_id=pr.id, state=PullRequestEventState.APPROVED.value, created_at=t3 - ) - approval_2 = get_pull_request_event( - pull_request_id=pr.id, state=PullRequestEventState.APPROVED.value, created_at=t4 - ) - performance = pr_service.get_pr_performance( - pr, [changes_requested_1, approval, approval_2] - ) - - assert performance.rework_time == (t3 - t2).total_seconds() - - -def test_pr_performance_returns_rework_time_for_open_prs(): - pr_service = CodeETLAnalyticsService() - t1 = time_now() - t2 = t1 + timedelta(hours=1) - t3 = t2 + timedelta(hours=1) - t4 = t3 + timedelta(hours=1) - pr = get_pull_request(state=PullRequestState.OPEN, created_at=t1, updated_at=t1) - changes_requested_1 = get_pull_request_event( - pull_request_id=pr.id, - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t2, - ) - approval = get_pull_request_event( - pull_request_id=pr.id, state=PullRequestEventState.APPROVED.value, created_at=t3 - ) - performance = pr_service.get_pr_performance(pr, [changes_requested_1, approval]) - - assert performance.rework_time == (t3 - t2).total_seconds() - - -def test_pr_performance_returns_rework_time_minus1_for_non_approved_prs(): - pr_service = CodeETLAnalyticsService() - t1 = time_now() - t2 = t1 + timedelta(hours=1) - t3 = t2 + timedelta(hours=1) - t4 = t3 + timedelta(hours=1) - pr = get_pull_request(state=PullRequestState.OPEN, created_at=t1, updated_at=t1) - changes_requested_1 = get_pull_request_event( - pull_request_id=pr.id, - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t2, - ) - performance = pr_service.get_pr_performance(pr, [changes_requested_1]) - - assert performance.rework_time == -1 - - -def test_pr_performance_returns_rework_time_minus1_for_merged_prs_without_reviews(): - pr_service = CodeETLAnalyticsService() - t1 = time_now() - pr = get_pull_request( - state=PullRequestState.MERGED, state_changed_at=t1, created_at=t1, updated_at=t1 - ) - performance = pr_service.get_pr_performance(pr, []) - - assert performance.rework_time == -1 - - -def test_pr_performance_returns_cycle_time_for_merged_pr(): - pr_service = CodeETLAnalyticsService() - t1 = time_now() - t2 = t1 + timedelta(days=1) - pr = get_pull_request( - state=PullRequestState.MERGED, state_changed_at=t2, created_at=t1, updated_at=t2 - ) - performance = pr_service.get_pr_performance(pr, []) - - assert performance.cycle_time == 86400 - - -def test_pr_performance_returns_cycle_time_minus1_for_non_merged_pr(): - pr_service = CodeETLAnalyticsService() - pr = get_pull_request() - performance = pr_service.get_pr_performance(pr, []) - - assert performance.cycle_time == -1 - - -def test_pr_rework_cycles_returns_zero_cycles_when_pr_approved(): - pr_service = CodeETLAnalyticsService() - pr = get_pull_request(reviewers=["dhruv", "jayant"]) - t1 = time_now() - t2 = t1 + timedelta(seconds=1) - commit = get_pull_request_commit(pr_id=pr.id, created_at=t1) - reviews = [get_pull_request_event(reviewer="dhruv", created_at=t2)] - assert pr_service.get_rework_cycles(pr, reviews, [commit]) == 0 - - -def test_rework_cycles_returns_1_cycle_if_some_rework_done(): - pr = get_pull_request(reviewers=["dhruv", "jayant"]) - t1 = time_now() - t2 = t1 + timedelta(seconds=1) - t3 = t1 + timedelta(seconds=2) - review_1 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t1, - ) - commit = get_pull_request_commit(pr_id=pr.id, created_at=t2) - review_2 = get_pull_request_event( - pull_request_id=pr.id, reviewer="dhruv", created_at=t3 - ) - pr_service = CodeETLAnalyticsService() - assert pr_service.get_rework_cycles(pr, [review_1, review_2], [commit]) == 1 - - -def test_rework_cycles_returns_2_cycles_if_there_were_comments_between_commit_batch(): - pr = get_pull_request(reviewers=["dhruv", "jayant"]) - t1 = time_now() - t2 = t1 + timedelta(seconds=1) - t3 = t1 + timedelta(seconds=2) - t4 = t1 + timedelta(seconds=3) - t5 = t1 + timedelta(seconds=4) - review_1 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t1, - ) - commit_1 = get_pull_request_commit(pr_id=pr.id, created_at=t2) - review_2 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t3, - ) - commit_2 = get_pull_request_commit(pr_id=pr.id, created_at=t4) - review_3 = get_pull_request_event( - pull_request_id=pr.id, reviewer="dhruv", created_at=t5 - ) - pr_service = CodeETLAnalyticsService() - assert ( - pr_service.get_rework_cycles( - pr, [review_1, review_2, review_3], [commit_1, commit_2] - ) - == 2 - ) - - -def test_rework_cycles_returns_1_cycle_despite_multiple_commits(): - pr = get_pull_request(reviewers=["dhruv", "jayant"]) - t1 = time_now() - t2 = t1 + timedelta(seconds=1) - t3 = t1 + timedelta(seconds=2) - review_1 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t1, - ) - commit_1 = get_pull_request_commit(pr_id=pr.id, created_at=t2) - commit_2 = get_pull_request_commit(pr_id=pr.id, created_at=t2) - commit_3 = get_pull_request_commit(pr_id=pr.id, created_at=t2) - review_2 = get_pull_request_event( - pull_request_id=pr.id, reviewer="dhruv", created_at=t3 - ) - pr_service = CodeETLAnalyticsService() - assert ( - pr_service.get_rework_cycles( - pr, [review_1, review_2], [commit_1, commit_2, commit_3] - ) - == 1 - ) - - -def test_rework_cycles_returns_2_cycles_despite_multiple_comments(): - pr = get_pull_request(reviewers=["dhruv", "jayant"]) - t1 = time_now() - t2 = t1 + timedelta(seconds=1) - t3 = t1 + timedelta(seconds=2) - t4 = t1 + timedelta(seconds=3) - t5 = t1 + timedelta(seconds=4) - review_1 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.COMMENTED.value, - created_at=t1, - ) - review_2 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.COMMENTED.value, - created_at=t1, - ) - review_3 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t1, - ) - commit_1 = get_pull_request_commit(pr_id=pr.id, created_at=t2) - review_4 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t3, - ) - commit_2 = get_pull_request_commit(pr_id=pr.id, created_at=t4) - review_5 = get_pull_request_event( - pull_request_id=pr.id, reviewer="dhruv", created_at=t5 - ) - review_6 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.COMMENTED.value, - created_at=t5, - ) - pr_service = CodeETLAnalyticsService() - assert ( - pr_service.get_rework_cycles( - pr, - [review_1, review_2, review_3, review_4, review_5, review_6], - [commit_1, commit_2], - ) - == 2 - ) - - -def test_rework_cycles_doesnt_count_commits_post_first_approval(): - pr = get_pull_request(reviewers=["dhruv", "jayant"]) - t1 = time_now() - t2 = t1 + timedelta(seconds=1) - t3 = t1 + timedelta(seconds=2) - t4 = t1 + timedelta(seconds=3) - t5 = t1 + timedelta(seconds=4) - t6 = t1 + timedelta(seconds=5) - review_1 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t1, - ) - commit_1 = get_pull_request_commit(pr_id=pr.id, created_at=t2) - review_2 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.COMMENTED.value, - created_at=t3, - ) - commit_2 = get_pull_request_commit(pr_id=pr.id, created_at=t4) - review_3 = get_pull_request_event( - pull_request_id=pr.id, reviewer="dhruv", created_at=t5 - ) - commit_3 = get_pull_request_commit(pr_id=pr.id, created_at=t6) - commit_4 = get_pull_request_commit(pr_id=pr.id, created_at=t6) - pr_service = CodeETLAnalyticsService() - assert ( - pr_service.get_rework_cycles( - pr, [review_1, review_2, review_3], [commit_1, commit_2, commit_3, commit_4] - ) - == 2 - ) - - -def test_rework_cycles_returns_0_for_unapproved_pr(): - pr = get_pull_request(reviewers=["dhruv", "jayant"]) - t1 = time_now() - t2 = t1 + timedelta(seconds=1) - t3 = t1 + timedelta(seconds=2) - t4 = t1 + timedelta(seconds=3) - t5 = t1 + timedelta(seconds=4) - t6 = t1 + timedelta(seconds=5) - review_1 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t1, - ) - commit_1 = get_pull_request_commit(pr_id=pr.id, created_at=t2) - review_2 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t3, - ) - commit_2 = get_pull_request_commit(pr_id=pr.id, created_at=t4) - review_3 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t5, - ) - commit_3 = get_pull_request_commit(pr_id=pr.id, created_at=t6) - commit_4 = get_pull_request_commit(pr_id=pr.id, created_at=t6) - pr_service = CodeETLAnalyticsService() - assert ( - pr_service.get_rework_cycles( - pr, [review_1, review_2, review_3], [commit_1, commit_2, commit_3, commit_4] - ) - == 0 - ) - - -def test_rework_cycles_returs_0_for_non_reviewer_comments(): - pr = get_pull_request(reviewers=["dhruv", "jayant"]) - t1 = time_now() - t2 = t1 + timedelta(seconds=1) - t3 = t1 + timedelta(seconds=2) - t4 = t1 + timedelta(seconds=3) - t5 = t1 + timedelta(seconds=4) - t6 = t1 + timedelta(seconds=5) - review_1 = get_pull_request_event( - pull_request_id=pr.id, - state=PullRequestEventState.COMMENTED.value, - created_at=t1, - ) - commit_1 = get_pull_request_commit(pr_id=pr.id, created_at=t2) - review_2 = get_pull_request_event( - pull_request_id=pr.id, - state=PullRequestEventState.COMMENTED.value, - created_at=t3, - ) - commit_2 = get_pull_request_commit(pr_id=pr.id, created_at=t4) - review_3 = get_pull_request_event( - pull_request_id=pr.id, - state=PullRequestEventState.COMMENTED.value, - created_at=t5, - ) - commit_3 = get_pull_request_commit(pr_id=pr.id, created_at=t6) - review_4 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.APPROVED.value, - created_at=t5, - ) - pr_service = CodeETLAnalyticsService() - assert ( - pr_service.get_rework_cycles( - pr, [review_1, review_2, review_3, review_4], [commit_1, commit_2, commit_3] - ) - == 0 - ) - - -def test_rework_cycles_returs_1_for_multiple_approvals(): - pr = get_pull_request(reviewers=["dhruv", "jayant"]) - t1 = time_now() - t2 = t1 + timedelta(seconds=1) - t3 = t1 + timedelta(seconds=2) - t4 = t1 + timedelta(seconds=3) - t5 = t1 + timedelta(seconds=4) - t6 = t1 + timedelta(seconds=5) - t7 = t1 + timedelta(seconds=6) - review_1 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.CHANGES_REQUESTED.value, - created_at=t1, - ) - commit_1 = get_pull_request_commit(pr_id=pr.id, created_at=t2) - review_2 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.APPROVED.value, - created_at=t3, - ) - commit_2 = get_pull_request_commit(pr_id=pr.id, created_at=t4) - review_3 = get_pull_request_event( - pull_request_id=pr.id, - state=PullRequestEventState.COMMENTED.value, - created_at=t5, - ) - commit_3 = get_pull_request_commit(pr_id=pr.id, created_at=t6) - review_4 = get_pull_request_event( - pull_request_id=pr.id, - reviewer="dhruv", - state=PullRequestEventState.APPROVED.value, - created_at=t7, - ) - pr_service = CodeETLAnalyticsService() - assert ( - pr_service.get_rework_cycles( - pr, [review_1, review_2, review_3, review_4], [commit_1, commit_2, commit_3] - ) - == 1 - ) diff --git a/apiserver/tests/service/code/sync/test_etl_github_handler.py b/apiserver/tests/service/code/sync/test_etl_github_handler.py deleted file mode 100644 index 287706717..000000000 --- a/apiserver/tests/service/code/sync/test_etl_github_handler.py +++ /dev/null @@ -1,353 +0,0 @@ -from datetime import datetime - -import pytz - -from dora.service.code.sync.etl_github_handler import GithubETLHandler -from dora.store.models.code import PullRequestState -from dora.utils.string import uuid4_str -from tests.factories.models import ( - get_pull_request, - get_pull_request_commit, - get_pull_request_event, -) -from tests.factories.models.exapi.github import ( - get_github_commit_dict, - get_github_pull_request_review, - get_github_pull_request, -) -from tests.utilities import compare_objects_as_dicts - -ORG_ID = uuid4_str() - - -def test__to_pr_model_given_a_github_pr_returns_new_pr_model(): - repo_id = uuid4_str() - number = 123 - user_login = "abc" - merged_at = datetime(2022, 6, 29, 10, 53, 15, tzinfo=pytz.UTC) - head_branch = "feature" - base_branch = "main" - title = "random_title" - review_comments = 3 - merge_commit_sha = "123456789098765" - - github_pull_request = get_github_pull_request( - number=number, - merged_at=merged_at, - head_ref=head_branch, - base_ref=base_branch, - user_login=user_login, - merge_commit_sha=merge_commit_sha, - commits=3, - additions=10, - deletions=5, - changed_files=2, - ) - - github_etl_handler = GithubETLHandler(ORG_ID, None, None, None, None) - pr_model = github_etl_handler._to_pr_model( - pr=github_pull_request, - pr_model=None, - repo_id=repo_id, - review_comments=review_comments, - ) - - expected_pr_model = get_pull_request( - repo_id=repo_id, - number=str(number), - author=str(user_login), - state=PullRequestState.MERGED, - title=title, - head_branch=head_branch, - base_branch=base_branch, - provider="github", - requested_reviews=[], - data=github_pull_request.raw_data, - state_changed_at=merged_at, - meta={ - "code_stats": { - "commits": 3, - "additions": 10, - "deletions": 5, - "changed_files": 2, - "comments": review_comments, - }, - "user_profile": { - "username": user_login, - }, - }, - reviewers=[], - merge_commit_sha=merge_commit_sha, - ) - # Ignoring the following fields as they are generated as side effects and are not part of the actual data - # reviewers, rework_time, first_commit_to_open, first_response_time, lead_time, merge_time, merge_to_deploy, cycle_time - assert ( - compare_objects_as_dicts( - pr_model, - expected_pr_model, - [ - "id", - "created_at", - "updated_at", - "reviewers", - "rework_time", - "first_commit_to_open", - "first_response_time", - "lead_time", - "merge_time", - "merge_to_deploy", - "cycle_time", - ], - ) - is True - ) - - -def test__to_pr_model_given_a_github_pr_and_db_pr_returns_updated_pr_model(): - repo_id = uuid4_str() - number = 123 - user_login = "abc" - merged_at = datetime(2022, 6, 29, 10, 53, 15, tzinfo=pytz.UTC) - head_branch = "feature" - base_branch = "main" - title = "random_title" - review_comments = 3 - merge_commit_sha = "123456789098765" - - github_pull_request = get_github_pull_request( - number=number, - merged_at=merged_at, - head_ref=head_branch, - base_ref=base_branch, - user_login=user_login, - merge_commit_sha=merge_commit_sha, - commits=3, - additions=10, - deletions=5, - changed_files=2, - ) - - given_pr_model = get_pull_request( - repo_id=repo_id, - number=str(number), - provider="github", - ) - - github_etl_handler = GithubETLHandler(ORG_ID, None, None, None, None) - pr_model = github_etl_handler._to_pr_model( - pr=github_pull_request, - pr_model=given_pr_model, - repo_id=repo_id, - review_comments=review_comments, - ) - - expected_pr_model = get_pull_request( - id=given_pr_model.id, - repo_id=repo_id, - number=str(number), - author=str(user_login), - state=PullRequestState.MERGED, - title=title, - head_branch=head_branch, - base_branch=base_branch, - provider="github", - requested_reviews=[], - data=github_pull_request.raw_data, - state_changed_at=merged_at, - meta={ - "code_stats": { - "commits": 3, - "additions": 10, - "deletions": 5, - "changed_files": 2, - "comments": review_comments, - }, - "user_profile": { - "username": user_login, - }, - }, - reviewers=[], - merge_commit_sha=merge_commit_sha, - ) - # Ignoring the following fields as they are generated as side effects and are not part of the actual data - # reviewers, rework_time, first_commit_to_open, first_response_time, lead_time, merge_time, merge_to_deploy, cycle_time - assert ( - compare_objects_as_dicts( - pr_model, - expected_pr_model, - [ - "created_at", - "updated_at", - "reviewers", - "rework_time", - "first_commit_to_open", - "first_response_time", - "lead_time", - "merge_time", - "merge_to_deploy", - "cycle_time", - ], - ) - is True - ) - - -def test__to_pr_events_given_an_empty_list_of_events_returns_an_empty_list(): - pr_model = get_pull_request() - assert GithubETLHandler._to_pr_events([], pr_model, []) == [] - - -def test__to_pr_events_given_a_list_of_only_new_events_returns_a_list_of_pr_events(): - pr_model = get_pull_request() - event1 = get_github_pull_request_review() - event2 = get_github_pull_request_review() - events = [event1, event2] - - pr_events = GithubETLHandler._to_pr_events(events, pr_model, []) - - expected_pr_events = [ - get_pull_request_event( - pull_request_id=str(pr_model.id), - org_repo_id=pr_model.repo_id, - data=event1.raw_data, - created_at=event1.submitted_at, - type="REVIEW", - idempotency_key=event1.id, - reviewer=event1.user_login, - ), - get_pull_request_event( - pull_request_id=str(pr_model.id), - org_repo_id=pr_model.repo_id, - data=event2.raw_data, - created_at=event2.submitted_at, - type="REVIEW", - idempotency_key=event2.id, - reviewer=event2.user_login, - ), - ] - - for event, expected_event in zip(pr_events, expected_pr_events): - assert compare_objects_as_dicts(event, expected_event, ["id"]) is True - - -def test__to_pr_events_given_a_list_of_new_events_and_old_events_returns_a_list_of_pr_events(): - pr_model = get_pull_request() - event1 = get_github_pull_request_review() - event2 = get_github_pull_request_review() - events = [event1, event2] - - old_event = get_pull_request_event( - pull_request_id=str(pr_model.id), - org_repo_id=pr_model.repo_id, - data=event1.raw_data, - created_at=event1.submitted_at, - type="REVIEW", - idempotency_key=event1.id, - reviewer=event1.user_login, - ) - - pr_events = GithubETLHandler._to_pr_events(events, pr_model, [old_event]) - - expected_pr_events = [ - old_event, - get_pull_request_event( - pull_request_id=str(pr_model.id), - org_repo_id=pr_model.repo_id, - data=event2.raw_data, - created_at=event2.submitted_at, - type="REVIEW", - idempotency_key=event2.id, - reviewer=event2.user_login, - ), - ] - - for event, expected_event in zip(pr_events, expected_pr_events): - assert compare_objects_as_dicts(event, expected_event, ["id"]) is True - - -def test__to_pr_commits_given_an_empty_list_of_commits_returns_an_empty_list(): - pr_model = get_pull_request() - github_etl_handler = GithubETLHandler(ORG_ID, None, None, None, None) - assert github_etl_handler._to_pr_commits([], pr_model) == [] - - -def test__to_pr_commits_given_a_list_of_commits_returns_a_list_of_pr_commits(): - pr_model = get_pull_request() - common_url = "random_url" - common_message = "random_message" - sha1 = "123456789098765" - author1 = "author_abc" - created_at1 = "2022-06-29T10:53:15Z" - commit1 = get_github_commit_dict( - sha=sha1, - author_login=author1, - created_at=created_at1, - url=common_url, - message=common_message, - ) - sha2 = "987654321234567" - author2 = "author_xyz" - created_at2 = "2022-06-29T12:53:15Z" - commit2 = get_github_commit_dict( - sha=sha2, - author_login=author2, - created_at=created_at2, - url=common_url, - message=common_message, - ) - sha3 = "543216789098765" - author3 = "author_abc" - created_at3 = "2022-06-29T15:53:15Z" - commit3 = get_github_commit_dict( - sha=sha3, - author_login=author3, - created_at=created_at3, - url=common_url, - message=common_message, - ) - - commits = [commit1, commit2, commit3] - github_etl_handler = GithubETLHandler(ORG_ID, None, None, None, None) - pr_commits = github_etl_handler._to_pr_commits(commits, pr_model) - - expected_pr_commits = [ - get_pull_request_commit( - pr_id=str(pr_model.id), - org_repo_id=pr_model.repo_id, - hash=sha1, - author=author1, - url=common_url, - message=common_message, - created_at=datetime(2022, 6, 29, 10, 53, 15, tzinfo=pytz.UTC), - data=commit1, - ), - get_pull_request_commit( - pr_id=str(pr_model.id), - org_repo_id=pr_model.repo_id, - hash=sha2, - author=author2, - url=common_url, - message=common_message, - created_at=datetime(2022, 6, 29, 12, 53, 15, tzinfo=pytz.UTC), - data=commit2, - ), - get_pull_request_commit( - pr_id=str(pr_model.id), - org_repo_id=pr_model.repo_id, - hash=sha3, - author=author3, - url=common_url, - message=common_message, - created_at=datetime(2022, 6, 29, 15, 53, 15, tzinfo=pytz.UTC), - data=commit3, - ), - ] - - for commit, expected_commit in zip(pr_commits, expected_pr_commits): - assert compare_objects_as_dicts(commit, expected_commit) is True - - -def test__dt_from_github_dt_string_given_date_string_returns_correct_datetime(): - date_string = "2024-04-18T10:53:15Z" - expected = datetime(2024, 4, 18, 10, 53, 15, tzinfo=pytz.UTC) - assert GithubETLHandler._dt_from_github_dt_string(date_string) == expected diff --git a/apiserver/tests/service/code/test_lead_time_service.py b/apiserver/tests/service/code/test_lead_time_service.py deleted file mode 100644 index 48b589e8b..000000000 --- a/apiserver/tests/service/code/test_lead_time_service.py +++ /dev/null @@ -1,198 +0,0 @@ -from dora.utils.string import uuid4_str -from tests.utilities import compare_objects_as_dicts -from dora.service.code.lead_time import LeadTimeService -from dora.service.code.models.lead_time import LeadTimeMetrics - - -class FakeCodeRepoService: - pass - - -class FakeDeploymentsService: - pass - - -def test_get_avg_time_for_multiple_lead_time_metrics_returns_correct_average(): - lead_time_metrics = [ - LeadTimeMetrics(first_response_time=1, pr_count=1), - LeadTimeMetrics(first_response_time=2, pr_count=1), - ] - field = "first_response_time" - - lead_time_service = LeadTimeService(FakeCodeRepoService, FakeDeploymentsService) - - result = lead_time_service._get_avg_time(lead_time_metrics, field) - assert result == 1.5 - - -def test_get_avg_time_for_different_lead_time_metrics_given_returns_correct_average(): - lead_time_metrics = [ - LeadTimeMetrics(first_response_time=1, pr_count=1), - LeadTimeMetrics(first_response_time=0, pr_count=0), - LeadTimeMetrics(first_response_time=3, pr_count=1), - ] - field = "first_response_time" - - lead_time_service = LeadTimeService(FakeCodeRepoService, FakeDeploymentsService) - - result = lead_time_service._get_avg_time(lead_time_metrics, field) - assert result == 2 - - -def test_get_avg_time_for_no_lead_time_metrics_returns_zero(): - lead_time_metrics = [] - field = "first_response_time" - - lead_time_service = LeadTimeService(FakeCodeRepoService, FakeDeploymentsService) - - result = lead_time_service._get_avg_time(lead_time_metrics, field) - assert result == 0 - - -def test_get_avg_time_for_empty_lead_time_metrics_returns_zero(): - lead_time_metrics = [LeadTimeMetrics(), LeadTimeMetrics()] - field = "first_response_time" - - lead_time_service = LeadTimeService(FakeCodeRepoService, FakeDeploymentsService) - - result = lead_time_service._get_avg_time(lead_time_metrics, field) - assert result == 0 - - -def test_get_weighted_avg_lead_time_metrics_returns_correct_average(): - lead_time_metrics = [ - LeadTimeMetrics( - first_commit_to_open=1, - first_response_time=4, - rework_time=10, - merge_time=1, - merge_to_deploy=1, - pr_count=1, - ), - LeadTimeMetrics( - first_commit_to_open=2, - first_response_time=3, - rework_time=10, - merge_time=1, - merge_to_deploy=1, - pr_count=2, - ), - LeadTimeMetrics( - first_commit_to_open=3, - first_response_time=2, - rework_time=10, - merge_time=1, - merge_to_deploy=1, - pr_count=3, - ), - LeadTimeMetrics( - first_commit_to_open=4, - first_response_time=1, - rework_time=10, - merge_time=1, - merge_to_deploy=1, - pr_count=4, - ), - ] - - lead_time_service = LeadTimeService(FakeCodeRepoService, FakeDeploymentsService) - - result = lead_time_service._get_weighted_avg_lead_time_metrics(lead_time_metrics) - - expected = LeadTimeMetrics( - first_commit_to_open=3, - first_response_time=2, - rework_time=10, - merge_time=1, - merge_to_deploy=1, - pr_count=10, - ) - assert compare_objects_as_dicts(result, expected) - - -def test_get_teams_avg_lead_time_metrics_returns_correct_values(): - - team_1 = uuid4_str() - team_2 = uuid4_str() - - team_lead_time_metrics = { - team_1: [ - LeadTimeMetrics( - first_commit_to_open=1, - first_response_time=4, - rework_time=10, - merge_time=1, - merge_to_deploy=1, - pr_count=1, - ), - LeadTimeMetrics( - first_commit_to_open=2, - first_response_time=3, - rework_time=10, - merge_time=1, - merge_to_deploy=1, - pr_count=2, - ), - LeadTimeMetrics( - first_commit_to_open=3, - first_response_time=2, - rework_time=10, - merge_time=1, - merge_to_deploy=1, - pr_count=3, - ), - LeadTimeMetrics( - first_commit_to_open=4, - first_response_time=1, - rework_time=10, - merge_time=1, - merge_to_deploy=1, - pr_count=4, - ), - ], - team_2: [ - LeadTimeMetrics( - first_commit_to_open=1, - first_response_time=4, - rework_time=10, - merge_time=1, - merge_to_deploy=1, - pr_count=1, - ), - LeadTimeMetrics( - first_commit_to_open=2, - first_response_time=3, - rework_time=10, - merge_time=1, - merge_to_deploy=1, - pr_count=2, - ), - ], - } - - lead_time_service = LeadTimeService(FakeCodeRepoService, FakeDeploymentsService) - - result = lead_time_service.get_avg_lead_time_metrics_from_map( - team_lead_time_metrics - ) - - expected = { - team_1: LeadTimeMetrics( - first_commit_to_open=3, - first_response_time=2, - rework_time=10, - merge_time=1, - merge_to_deploy=1, - pr_count=10, - ), - team_2: LeadTimeMetrics( - first_commit_to_open=5 / 3, - first_response_time=10 / 3, - rework_time=10, - merge_time=1, - merge_to_deploy=1, - pr_count=3, - ), - } - assert compare_objects_as_dicts(result[team_1], expected[team_1]) - assert compare_objects_as_dicts(result[team_2], expected[team_2]) diff --git a/apiserver/tests/service/deployments/__init__.py b/apiserver/tests/service/deployments/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/tests/service/deployments/test_deployment_frequency.py b/apiserver/tests/service/deployments/test_deployment_frequency.py deleted file mode 100644 index c84df8680..000000000 --- a/apiserver/tests/service/deployments/test_deployment_frequency.py +++ /dev/null @@ -1,181 +0,0 @@ -from datetime import datetime, timedelta - -import pytz -from dora.service.deployments.analytics import DeploymentAnalyticsService -from dora.utils.time import Interval -from tests.factories.models.code import get_deployment, get_deployment_frequency_metrics - -first_week_2024 = datetime(2024, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) -second_week_2024 = datetime(2024, 1, 8, 0, 0, 0, tzinfo=pytz.UTC) -third_week_2024 = datetime(2024, 1, 15, 0, 0, 0, tzinfo=pytz.UTC) -fourth_week_2024 = datetime(2024, 1, 22, 0, 0, 0, tzinfo=pytz.UTC) - - -def test_deployment_frequency_for_no_deployments(): - - from_time = first_week_2024 + timedelta(days=1) - to_time = third_week_2024 + timedelta(days=2) - - deployment_analytics_service = DeploymentAnalyticsService(None, None) - - assert ( - deployment_analytics_service._get_deployment_frequency_metrics( - [], Interval(from_time, to_time) - ) - == get_deployment_frequency_metrics() - ) - - -def test_deployment_frequency_for_deployments_across_days(): - - from_time = first_week_2024 + timedelta(days=1) - to_time = first_week_2024 + timedelta(days=4) - - deployment_1 = get_deployment(conducted_at=from_time + timedelta(hours=12)) - deployment_2 = get_deployment(conducted_at=from_time + timedelta(days=1)) - deployment_3 = get_deployment(conducted_at=from_time + timedelta(days=2)) - - deployment_outside_interval = get_deployment( - conducted_at=to_time + timedelta(days=20) - ) - - deployment_analytics_service = DeploymentAnalyticsService(None, None) - - assert deployment_analytics_service._get_deployment_frequency_metrics( - [deployment_1, deployment_2, deployment_3, deployment_outside_interval], - Interval(from_time, to_time), - ) == get_deployment_frequency_metrics(3, 0, 3, 3) - - -def test_deployment_frequency_for_deployments_across_weeks(): - - from_time = first_week_2024 + timedelta(days=1) - to_time = fourth_week_2024 + timedelta(days=1) - - # Week 1 - - deployment_1 = get_deployment(conducted_at=from_time + timedelta(hours=12)) - deployment_2 = get_deployment(conducted_at=from_time + timedelta(hours=24)) - - # Week 3 - deployment_3 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=4)) - deployment_4 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=2)) - deployment_5 = get_deployment(conducted_at=fourth_week_2024 - timedelta(hours=6)) - deployment_6 = get_deployment(conducted_at=fourth_week_2024 - timedelta(minutes=30)) - - deployment_analytics_service = DeploymentAnalyticsService(None, None) - - assert deployment_analytics_service._get_deployment_frequency_metrics( - [ - deployment_1, - deployment_2, - deployment_3, - deployment_4, - deployment_5, - deployment_6, - ], - Interval(from_time, to_time), - ) == get_deployment_frequency_metrics(6, 0, 1, 6) - - -def test_deployment_frequency_for_deployments_across_months(): - - from_time = first_week_2024 + timedelta(days=1) - to_time = datetime(2024, 3, 31, 0, 0, 0, tzinfo=pytz.UTC) - - second_month_2024 = datetime(2024, 2, 1, 0, 0, 0, tzinfo=pytz.UTC) - - print((to_time - from_time).days) - - # Month 1 - - deployment_1 = get_deployment(conducted_at=from_time + timedelta(hours=12)) - deployment_2 = get_deployment(conducted_at=from_time + timedelta(hours=24)) - deployment_3 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=4)) - deployment_4 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=2)) - deployment_5 = get_deployment(conducted_at=fourth_week_2024 - timedelta(hours=6)) - deployment_6 = get_deployment(conducted_at=fourth_week_2024 - timedelta(minutes=30)) - - # Month 2 - - deployment_7 = get_deployment(conducted_at=second_month_2024 + timedelta(days=3)) - deployment_8 = get_deployment(conducted_at=second_month_2024 + timedelta(days=2)) - deployment_9 = get_deployment( - conducted_at=second_month_2024 + timedelta(minutes=30) - ) - - # Month 3 - - deployment_10 = get_deployment(conducted_at=to_time - timedelta(days=3)) - deployment_11 = get_deployment(conducted_at=to_time - timedelta(days=2)) - deployment_12 = get_deployment(conducted_at=to_time - timedelta(days=1)) - deployment_13 = get_deployment(conducted_at=to_time - timedelta(days=1)) - - deployment_analytics_service = DeploymentAnalyticsService(None, None) - - assert deployment_analytics_service._get_deployment_frequency_metrics( - [ - deployment_1, - deployment_2, - deployment_3, - deployment_4, - deployment_5, - deployment_6, - deployment_7, - deployment_8, - deployment_9, - deployment_10, - deployment_11, - deployment_12, - deployment_13, - ], - Interval(from_time, to_time), - ) == get_deployment_frequency_metrics(13, 0, 1, 4) - - -def test_weekly_deployment_frequency_trends_for_no_deployments(): - - from_time = first_week_2024 + timedelta(days=1) - to_time = third_week_2024 + timedelta(days=2) - - deployment_analytics_service = DeploymentAnalyticsService(None, None) - - assert deployment_analytics_service._get_weekly_deployment_frequency_trends( - [], Interval(from_time, to_time) - ) == {first_week_2024: 0, second_week_2024: 0, third_week_2024: 0} - - -def test_weekly_deployment_frequency_trends_for_deployments(): - - from_time = first_week_2024 + timedelta(days=1) - to_time = fourth_week_2024 + timedelta(days=1) - - # Week 1 - - deployment_1 = get_deployment(conducted_at=from_time + timedelta(hours=12)) - deployment_2 = get_deployment(conducted_at=from_time + timedelta(hours=24)) - - # Week 3 - deployment_3 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=4)) - deployment_4 = get_deployment(conducted_at=fourth_week_2024 - timedelta(days=2)) - deployment_5 = get_deployment(conducted_at=fourth_week_2024 - timedelta(hours=6)) - deployment_6 = get_deployment(conducted_at=fourth_week_2024 - timedelta(minutes=30)) - - deployment_analytics_service = DeploymentAnalyticsService(None, None) - - assert deployment_analytics_service._get_weekly_deployment_frequency_trends( - [ - deployment_1, - deployment_2, - deployment_3, - deployment_4, - deployment_5, - deployment_6, - ], - Interval(from_time, to_time), - ) == { - first_week_2024: 2, - second_week_2024: 0, - third_week_2024: 4, - fourth_week_2024: 0, - } diff --git a/apiserver/tests/service/deployments/test_deployment_pr_mapper.py b/apiserver/tests/service/deployments/test_deployment_pr_mapper.py deleted file mode 100644 index cdbf8f88d..000000000 --- a/apiserver/tests/service/deployments/test_deployment_pr_mapper.py +++ /dev/null @@ -1,183 +0,0 @@ -from datetime import timedelta - -from dora.service.deployments.deployment_pr_mapper import DeploymentPRMapperService -from dora.store.models.code import PullRequestState -from dora.utils.time import time_now -from tests.factories.models.code import get_pull_request, get_repo_workflow_run - - -def test_deployment_pr_mapper_picks_prs_directly_merged_to_head_branch(): - t = time_now() - dep_branch = "release" - pr_to_main = get_pull_request( - state=PullRequestState.MERGED, - head_branch="feature", - base_branch="main", - state_changed_at=t, - ) - pr_to_release = get_pull_request( - state=PullRequestState.MERGED, - head_branch="feature", - base_branch="release", - state_changed_at=t + timedelta(days=2), - ) - assert DeploymentPRMapperService().get_all_prs_deployed( - [pr_to_main, pr_to_release], - get_repo_workflow_run( - head_branch=dep_branch, conducted_at=t + timedelta(days=7) - ), - ) == [pr_to_release] - - -def test_deployment_pr_mapper_ignores_prs_not_related_to_head_branch_directly_or_indirectly(): - t = time_now() - dep_branch = "release2" - pr_to_main = get_pull_request( - state=PullRequestState.MERGED, - head_branch="feature", - base_branch="main", - state_changed_at=t + timedelta(days=1), - ) - pr_to_release = get_pull_request( - state=PullRequestState.MERGED, - head_branch="feature", - base_branch="release", - state_changed_at=t + timedelta(days=2), - ) - assert ( - DeploymentPRMapperService().get_all_prs_deployed( - [pr_to_main, pr_to_release], - get_repo_workflow_run( - head_branch=dep_branch, conducted_at=t + timedelta(days=7) - ), - ) - == [] - ) - - -def test_deployment_pr_mapper_picks_prs_on_the_path_to_head_branch(): - t = time_now() - dep_branch = "release" - pr_to_feature = get_pull_request( - state=PullRequestState.MERGED, - head_branch="custom_feature", - base_branch="feature", - ) - pr_to_main = get_pull_request( - state=PullRequestState.MERGED, - head_branch="feature", - base_branch="main", - state_changed_at=t + timedelta(days=2), - ) - pr_to_release = get_pull_request( - state=PullRequestState.MERGED, - head_branch="main", - base_branch="release", - state_changed_at=t + timedelta(days=4), - ) - assert sorted( - [ - x.id - for x in DeploymentPRMapperService().get_all_prs_deployed( - [pr_to_feature, pr_to_main, pr_to_release], - get_repo_workflow_run( - head_branch=dep_branch, conducted_at=t + timedelta(days=7) - ), - ) - ] - ) == sorted([x.id for x in [pr_to_main, pr_to_release, pr_to_feature]]) - - -def test_deployment_pr_mapper_doesnt_pick_any_pr_if_no_pr_merged_to_head_branch(): - t = time_now() - dep_branch = "release" - pr_to_main = get_pull_request( - state=PullRequestState.MERGED, - head_branch="feature", - base_branch="main", - state_changed_at=t, - ) - assert ( - DeploymentPRMapperService().get_all_prs_deployed( - [pr_to_main], - get_repo_workflow_run( - head_branch=dep_branch, conducted_at=t + timedelta(days=4) - ), - ) - == [] - ) - - -def test_deployment_pr_mapper_picks_only_merged_prs_not_open_or_closed(): - t = time_now() - dep_branch = "release" - pr_to_feature = get_pull_request( - state=PullRequestState.OPEN, - head_branch="custom_feature", - base_branch="feature", - created_at=t, - state_changed_at=None, - ) - pr_to_main = get_pull_request( - state=PullRequestState.MERGED, - head_branch="feature", - base_branch="main", - state_changed_at=t + timedelta(days=2), - ) - pr_to_release = get_pull_request( - state=PullRequestState.CLOSED, - head_branch="main", - base_branch="release", - state_changed_at=t + timedelta(days=3), - ) - assert ( - DeploymentPRMapperService().get_all_prs_deployed( - [pr_to_feature, pr_to_main, pr_to_release], - get_repo_workflow_run( - head_branch=dep_branch, conducted_at=t + timedelta(days=7) - ), - ) - == [] - ) - - -def test_deployment_pr_mapper_returns_empty_for_no_prs(): - dep_branch = "release" - assert ( - DeploymentPRMapperService().get_all_prs_deployed( - [], get_repo_workflow_run(head_branch=dep_branch) - ) - == [] - ) - - -def test_deployment_pr_mapper_ignores_sub_prs_merged_post_main_pr_merge(): - dep_branch = "release" - t = time_now() - first_feature_pr = get_pull_request( - state=PullRequestState.MERGED, - head_branch="feature", - base_branch="master", - state_changed_at=t, - ) - second_feature_pr = get_pull_request( - state=PullRequestState.MERGED, - head_branch="feature", - base_branch="master", - state_changed_at=t + timedelta(days=4), - ) - pr_to_release = get_pull_request( - state=PullRequestState.MERGED, - head_branch="master", - base_branch="release", - state_changed_at=t + timedelta(days=2), - ) - - prs = DeploymentPRMapperService().get_all_prs_deployed( - [first_feature_pr, second_feature_pr, pr_to_release], - get_repo_workflow_run( - head_branch=dep_branch, conducted_at=t + timedelta(days=5) - ), - ) - - assert sorted(prs) == sorted([first_feature_pr, pr_to_release]) diff --git a/apiserver/tests/service/workflows/__init__.py b/apiserver/tests/service/workflows/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/tests/service/workflows/sync/__init__.py b/apiserver/tests/service/workflows/sync/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apiserver/tests/service/workflows/sync/test_etl_github_actions_handler.py b/apiserver/tests/service/workflows/sync/test_etl_github_actions_handler.py deleted file mode 100644 index 19c457ed9..000000000 --- a/apiserver/tests/service/workflows/sync/test_etl_github_actions_handler.py +++ /dev/null @@ -1,110 +0,0 @@ -from dora.service.workflows.sync.etl_github_actions_handler import ( - GithubActionsETLHandler, -) -from dora.store.models.code import RepoWorkflowRunsStatus -from dora.utils.string import uuid4_str -from tests.factories.models import get_repo_workflow_run -from tests.factories.models.exapi.github import get_github_workflow_run_dict -from tests.utilities import compare_objects_as_dicts - - -def test__adapt_github_workflows_to_workflow_runs_given_new_workflow_run_return_new_run(): - class WorkflowRepoService: - def get_repo_workflow_run_by_provider_workflow_run_id(self, *args): - return None - - github_workflow_run = get_github_workflow_run_dict() - org_id = uuid4_str() - repo_id = uuid4_str() - gh_actions_etl_handler = GithubActionsETLHandler(org_id, None, WorkflowRepoService) - actual_workflow_run = ( - gh_actions_etl_handler._adapt_github_workflows_to_workflow_runs( - repo_id, github_workflow_run - ) - ) - - expected_workflow_run = get_repo_workflow_run( - repo_workflow_id=repo_id, - provider_workflow_run_id=str(github_workflow_run["id"]), - event_actor=github_workflow_run["actor"]["login"], - head_branch=github_workflow_run["head_branch"], - status=RepoWorkflowRunsStatus.SUCCESS, - conducted_at=gh_actions_etl_handler._get_datetime_from_gh_datetime( - github_workflow_run["run_started_at"] - ), - duration=gh_actions_etl_handler._get_repo_workflow_run_duration( - github_workflow_run - ), - meta=github_workflow_run, - html_url=github_workflow_run["html_url"], - ) - - assert compare_objects_as_dicts( - actual_workflow_run, expected_workflow_run, ["id", "created_at", "updated_at"] - ) - - -def test__adapt_github_workflows_to_workflow_runs_given_already_synced_workflow_run_returns_updated_run(): - github_workflow_run = get_github_workflow_run_dict() - org_id = uuid4_str() - repo_id = uuid4_str() - - repo_workflow_run_in_db = get_repo_workflow_run( - repo_workflow_id=repo_id, - provider_workflow_run_id=str(github_workflow_run["id"]), - ) - - class WorkflowRepoService: - def get_repo_workflow_run_by_provider_workflow_run_id(self, *args): - return repo_workflow_run_in_db - - gh_actions_etl_handler = GithubActionsETLHandler(org_id, None, WorkflowRepoService) - actual_workflow_run = ( - gh_actions_etl_handler._adapt_github_workflows_to_workflow_runs( - repo_id, github_workflow_run - ) - ) - - expected_workflow_run = get_repo_workflow_run( - id=repo_workflow_run_in_db.id, - repo_workflow_id=repo_id, - provider_workflow_run_id=str(github_workflow_run["id"]), - event_actor=github_workflow_run["actor"]["login"], - head_branch=github_workflow_run["head_branch"], - status=RepoWorkflowRunsStatus.SUCCESS, - conducted_at=gh_actions_etl_handler._get_datetime_from_gh_datetime( - github_workflow_run["run_started_at"] - ), - duration=gh_actions_etl_handler._get_repo_workflow_run_duration( - github_workflow_run - ), - meta=github_workflow_run, - html_url=github_workflow_run["html_url"], - ) - - assert compare_objects_as_dicts( - actual_workflow_run, expected_workflow_run, ["created_at", "updated_at"] - ) - - -def test__get_repo_workflow_run_duration_given_workflow_run_with_timings_returns_correct_duration(): - repo_workflow_run = get_github_workflow_run_dict( - run_started_at="2021-06-01T12:00:00Z", updated_at="2021-06-01T12:11:00Z" - ) - org_id = uuid4_str() - expected_duration = 660 - gh_actions_etl_handler = GithubActionsETLHandler(org_id, None, None) - actual_duration = gh_actions_etl_handler._get_repo_workflow_run_duration( - repo_workflow_run - ) - assert actual_duration == expected_duration - - -def test__get_repo_workflow_run_duration_given_workflow_run_without_timings_returns_none(): - repo_workflow_run = get_github_workflow_run_dict(run_started_at="", updated_at="") - org_id = uuid4_str() - gh_actions_etl_handler = GithubActionsETLHandler(org_id, None, None) - actual_duration = gh_actions_etl_handler._get_repo_workflow_run_duration( - repo_workflow_run - ) - assert actual_duration is None diff --git a/apiserver/tests/utilities.py b/apiserver/tests/utilities.py deleted file mode 100644 index 9a101f7c4..000000000 --- a/apiserver/tests/utilities.py +++ /dev/null @@ -1,20 +0,0 @@ -def compare_objects_as_dicts(ob_1, ob_2, ignored_keys=None): - """ - This method can be used to compare between two objects in tests while ignoring keys that are generated as side effects like uuids or autogenerated date time fields. - """ - if not ignored_keys: - ignored_keys = [] - - default_ignored_keys = ["_sa_instance_state"] - final_ignored_keys = set(ignored_keys + default_ignored_keys) - - for key in final_ignored_keys: - if key in ob_1.__dict__: - del ob_1.__dict__[key] - if key in ob_2.__dict__: - del ob_2.__dict__[key] - - if not ob_1.__dict__ == ob_2.__dict__: - print(ob_1.__dict__, "!=", ob_2.__dict__) - return False - return True diff --git a/apiserver/tests/utils/dict/test_get_average_of_dict_values.py b/apiserver/tests/utils/dict/test_get_average_of_dict_values.py deleted file mode 100644 index ff6b429d2..000000000 --- a/apiserver/tests/utils/dict/test_get_average_of_dict_values.py +++ /dev/null @@ -1,17 +0,0 @@ -from dora.utils.dict import get_average_of_dict_values - - -def test_empty_dict_returns_zero(): - assert get_average_of_dict_values({}) == 0 - - -def test_nulls_counted_as_zero(): - assert get_average_of_dict_values({"w1": 2, "w2": 4, "w3": None}) == 2 - - -def test_average_of_integers_with_integer_avg(): - assert get_average_of_dict_values({"w1": 2, "w2": 4, "w3": 6}) == 4 - - -def test_average_of_integers_with_decimal_avg_rounded_off(): - assert get_average_of_dict_values({"w1": 2, "w2": 4, "w3": 7}) == 4 diff --git a/apiserver/tests/utils/dict/test_get_key_to_count_map.py b/apiserver/tests/utils/dict/test_get_key_to_count_map.py deleted file mode 100644 index e9ace5512..000000000 --- a/apiserver/tests/utils/dict/test_get_key_to_count_map.py +++ /dev/null @@ -1,29 +0,0 @@ -from dora.utils.dict import get_key_to_count_map_from_key_to_list_map - - -def test_empty_dict_return_empty_dict(): - assert get_key_to_count_map_from_key_to_list_map({}) == {} - - -def test_dict_with_list_values(): - assert get_key_to_count_map_from_key_to_list_map( - {"a": [1, 2], "b": ["a", "p", "9"]} - ) == {"a": 2, "b": 3} - - -def test_dict_with_set_values(): - assert get_key_to_count_map_from_key_to_list_map( - {"a": {1, 2}, "b": {"a", "p", "9"}} - ) == {"a": 2, "b": 3} - - -def test_dict_with_non_set_or_list_values(): - assert get_key_to_count_map_from_key_to_list_map( - {"a": None, "b": 0, "c": "Ckk"} - ) == {"a": 0, "b": 0, "c": 0} - - -def test_dict_with_mixed_values(): - assert get_key_to_count_map_from_key_to_list_map( - {"a": None, "b": 0, "c": "Ckk", "e": [1], "g": {"A", "B"}} - ) == {"a": 0, "b": 0, "c": 0, "e": 1, "g": 2} diff --git a/apiserver/tests/utils/time/test_fill_missing_week_buckets.py b/apiserver/tests/utils/time/test_fill_missing_week_buckets.py deleted file mode 100644 index 48a2f8652..000000000 --- a/apiserver/tests/utils/time/test_fill_missing_week_buckets.py +++ /dev/null @@ -1,74 +0,0 @@ -from dataclasses import dataclass -from datetime import datetime - -import pytz - -from dora.utils.time import Interval, fill_missing_week_buckets - - -last_week_2022 = datetime(2022, 12, 26, 0, 0, 0, tzinfo=pytz.UTC) -first_week_2023 = datetime(2023, 1, 2, 0, 0, 0, tzinfo=pytz.UTC) -second_week_2023 = datetime(2023, 1, 9, 0, 0, 0, tzinfo=pytz.UTC) -third_week_2023 = datetime(2023, 1, 16, 0, 0, 0, tzinfo=pytz.UTC) -fourth_week_2023 = datetime(2023, 1, 23, 0, 0, 0, tzinfo=pytz.UTC) - - -@dataclass -class sample_class: - score: int = 10 - name: str = "MHQ" - - -def test_fill_missing_buckets_fills_missing_weeks_in_middle(): - interval = Interval(last_week_2022, fourth_week_2023) - assert fill_missing_week_buckets( - {last_week_2022: sample_class(1, ""), fourth_week_2023: sample_class(2, "")}, - interval, - ) == { - last_week_2022: sample_class(1, ""), - first_week_2023: None, - second_week_2023: None, - third_week_2023: None, - fourth_week_2023: sample_class(2, ""), - } - - -def test_fill_missing_buckets_fills_missing_weeks_in_past(): - interval = Interval(last_week_2022, fourth_week_2023) - assert fill_missing_week_buckets( - {third_week_2023: sample_class(1, ""), fourth_week_2023: sample_class(2, "")}, - interval, - ) == { - last_week_2022: None, - first_week_2023: None, - second_week_2023: None, - third_week_2023: sample_class(1, ""), - fourth_week_2023: sample_class(2, ""), - } - - -def test_fill_missing_buckets_fills_missing_weeks_in_future(): - interval = Interval(last_week_2022, fourth_week_2023) - assert fill_missing_week_buckets( - {last_week_2022: sample_class(1, ""), first_week_2023: sample_class(2, "")}, - interval, - ) == { - last_week_2022: sample_class(1, ""), - first_week_2023: sample_class(2, ""), - second_week_2023: None, - third_week_2023: None, - fourth_week_2023: None, - } - - -def test_fill_missing_buckets_fills_past_and_future_weeks_with_callable(): - interval = Interval(last_week_2022, fourth_week_2023) - assert fill_missing_week_buckets( - {first_week_2023: sample_class(2, "")}, interval, sample_class - ) == { - last_week_2022: sample_class(), - first_week_2023: sample_class(2, ""), - second_week_2023: sample_class(), - third_week_2023: sample_class(), - fourth_week_2023: sample_class(), - } diff --git a/apiserver/tests/utils/time/test_generate_expanded_buckets.py b/apiserver/tests/utils/time/test_generate_expanded_buckets.py deleted file mode 100644 index 95d3706f9..000000000 --- a/apiserver/tests/utils/time/test_generate_expanded_buckets.py +++ /dev/null @@ -1,288 +0,0 @@ -from collections import defaultdict -from dataclasses import dataclass -from datetime import datetime, timedelta -from typing import Dict, List, Any - -import pytest -import pytz - -from dora.utils.time import ( - Interval, - generate_expanded_buckets, - get_given_weeks_monday, - time_now, -) - - -@dataclass -class AnyObject: - state_changed_at: datetime - - -def test_incorrect_interval_raises_exception(): - object_list = [] - from_time = time_now() - to_time = from_time - timedelta(seconds=1) - - attribute = "state_changed_at" - with pytest.raises(AssertionError) as e: - buckets = generate_expanded_buckets( - object_list, Interval(from_time, to_time), attribute - ) - assert ( - str(e.value) - == f"from_time: {from_time.isoformat()} is greater than to_time: {to_time.isoformat()}" - ) - - -def test_missing_attribute_raise_exception(): - object_list = [AnyObject(time_now())] - from_time = time_now() - to_time = from_time + timedelta(seconds=1) - attribute = "updated_at" - with pytest.raises(AttributeError) as e: - buckets = generate_expanded_buckets( - object_list, Interval(from_time, to_time), attribute - ) - - -def test_incorrect_attribute_type_raise_exception(): - object_list = [AnyObject("hello")] - from_time = time_now() - to_time = from_time + timedelta(seconds=1) - attribute = "state_changed_at" - with pytest.raises(Exception) as e: - buckets = generate_expanded_buckets( - object_list, Interval(from_time, to_time), attribute - ) - assert ( - str(e.value) - == f"Type of datetime_attribute:{type(getattr(object_list[0], attribute))} is not datetime" - ) - - -def test_empty_data_generates_correct_buckets(): - object_list = [] - from_time = time_now() - timedelta(days=10) - to_time = from_time + timedelta(seconds=1) - attribute = "state_changed_at" - - ans_buckets = defaultdict(list) - - curr_date = get_given_weeks_monday(from_time) - - while curr_date < to_time: - ans_buckets[curr_date] = [] - curr_date = curr_date + timedelta(days=7) - - assert ans_buckets == generate_expanded_buckets( - object_list, Interval(from_time, to_time), attribute - ) - - -def test_data_generates_empty_middle_buckets(): - first_week_2023 = datetime(2023, 1, 2, 0, 0, 0, tzinfo=pytz.UTC) - second_week_2023 = datetime(2023, 1, 9, 0, 0, 0, tzinfo=pytz.UTC) - third_week_2023 = datetime(2023, 1, 16, 0, 0, 0, tzinfo=pytz.UTC) - - from_time = first_week_2023 + timedelta(days=1) - to_time = third_week_2023 + timedelta(days=5) - - obj1 = AnyObject(first_week_2023 + timedelta(days=2)) - obj2 = AnyObject(first_week_2023 + timedelta(days=3)) - obj3 = AnyObject(first_week_2023 + timedelta(days=4)) - obj4 = AnyObject(third_week_2023 + timedelta(days=2)) - obj5 = AnyObject(third_week_2023 + timedelta(days=3)) - obj6 = AnyObject(third_week_2023 + timedelta(days=4)) - object_list = [obj1, obj2, obj3, obj4, obj5, obj6] - - attribute = "state_changed_at" - - ans_buckets: Dict[datetime, List[Any]] = defaultdict(list) - - ans_buckets[first_week_2023] = [obj1, obj2, obj3] - ans_buckets[second_week_2023] = [] - ans_buckets[third_week_2023] = [obj4, obj5, obj6] - - curr_date = get_given_weeks_monday(from_time) - - assert ans_buckets == generate_expanded_buckets( - object_list, Interval(from_time, to_time), attribute - ) - - -def test_data_within_interval_generates_correctly_filled_buckets(): - first_week_2023 = datetime(2023, 1, 2, 0, 0, 0, tzinfo=pytz.UTC) - second_week_2023 = datetime(2023, 1, 9, 0, 0, 0, tzinfo=pytz.UTC) - third_week_2023 = datetime(2023, 1, 16, 0, 0, 0, tzinfo=pytz.UTC) - - from_time = first_week_2023 + timedelta(days=1) - to_time = third_week_2023 + timedelta(days=5) - - obj1 = AnyObject(first_week_2023 + timedelta(days=2)) - obj2 = AnyObject(first_week_2023 + timedelta(days=3)) - obj3 = AnyObject(first_week_2023 + timedelta(days=4)) - obj4 = AnyObject(second_week_2023) - obj5 = AnyObject(second_week_2023 + timedelta(days=6)) - obj6 = AnyObject(third_week_2023 + timedelta(days=4)) - obj7 = AnyObject(third_week_2023 + timedelta(days=2)) - obj8 = AnyObject(third_week_2023 + timedelta(days=3)) - obj9 = AnyObject(third_week_2023 + timedelta(days=4)) - object_list = [obj1, obj2, obj3, obj4, obj5, obj6, obj7, obj8, obj9] - - attribute = "state_changed_at" - - ans_buckets = defaultdict(list) - - ans_buckets[first_week_2023] = [obj1, obj2, obj3] - ans_buckets[second_week_2023] = [obj4, obj5] - ans_buckets[third_week_2023] = [obj6, obj7, obj8, obj9] - - assert ans_buckets == generate_expanded_buckets( - object_list, Interval(from_time, to_time), attribute - ) - - -def test_data_outside_interval_generates_correctly_filled_buckets(): - last_week_2022 = datetime(2022, 12, 26, 0, 0, 0, tzinfo=pytz.UTC) - first_week_2023 = datetime(2023, 1, 2, 0, 0, 0, tzinfo=pytz.UTC) - second_week_2023 = datetime(2023, 1, 9, 0, 0, 0, tzinfo=pytz.UTC) - third_week_2023 = datetime(2023, 1, 16, 0, 0, 0, tzinfo=pytz.UTC) - fourth_week_2023 = datetime(2023, 1, 23, 0, 0, 0, tzinfo=pytz.UTC) - - from_time = first_week_2023 + timedelta(days=1) - to_time = third_week_2023 + timedelta(days=5) - - obj1 = AnyObject(last_week_2022 + timedelta(days=2)) - obj2 = AnyObject(last_week_2022 + timedelta(days=3)) - obj3 = AnyObject(last_week_2022 + timedelta(days=4)) - obj4 = AnyObject(last_week_2022) - obj5 = AnyObject(second_week_2023 + timedelta(days=6)) - obj6 = AnyObject(fourth_week_2023 + timedelta(days=4)) - obj7 = AnyObject(fourth_week_2023 + timedelta(days=2)) - obj8 = AnyObject(fourth_week_2023 + timedelta(days=3)) - obj9 = AnyObject(fourth_week_2023 + timedelta(days=4)) - object_list = [obj1, obj2, obj3, obj4, obj5, obj6, obj7, obj8, obj9] - - attribute = "state_changed_at" - - ans_buckets = defaultdict(list) - - ans_buckets[last_week_2022] = [obj1, obj2, obj3, obj4] - ans_buckets[first_week_2023] = [] - ans_buckets[second_week_2023] = [obj5] - ans_buckets[third_week_2023] = [] - ans_buckets[fourth_week_2023] = [obj6, obj7, obj8, obj9] - - assert ans_buckets == generate_expanded_buckets( - object_list, Interval(from_time, to_time), attribute - ) - - -def test_daily_buckets_with_one_per_day(): - object_list = [ - AnyObject(datetime(2022, 1, 1, tzinfo=pytz.UTC)), - AnyObject(datetime(2022, 1, 2, tzinfo=pytz.UTC)), - AnyObject(datetime(2022, 1, 2, tzinfo=pytz.UTC)), - ] - from_time = datetime(2022, 1, 1, tzinfo=pytz.UTC) - to_time = datetime(2022, 1, 3, tzinfo=pytz.UTC) - attribute = "state_changed_at" - granularity = "daily" - ans_buckets = defaultdict(list) - ans_buckets[datetime.fromisoformat("2022-01-01T00:00:00+00:00")] = [object_list[0]] - ans_buckets[datetime.fromisoformat("2022-01-02T00:00:00+00:00")] = object_list[1:] - ans_buckets[datetime.fromisoformat("2022-01-03T00:00:00+00:00")] = [] - assert ans_buckets == generate_expanded_buckets( - object_list, Interval(from_time, to_time), attribute, granularity - ) - - -def test_daily_buckets_with_multiple_per_day(): - object_list_2 = [ - AnyObject(datetime(2022, 1, 1, tzinfo=pytz.UTC)), - AnyObject(datetime(2022, 1, 2, tzinfo=pytz.UTC)), - AnyObject(datetime(2022, 1, 2, 12, 30, tzinfo=pytz.UTC)), - ] - from_time_2 = datetime(2022, 1, 1, tzinfo=pytz.UTC) - to_time_2 = datetime(2022, 1, 3, tzinfo=pytz.UTC) - attribute = "state_changed_at" - granularity = "daily" - ans_buckets_2 = defaultdict(list) - ans_buckets_2[datetime.fromisoformat("2022-01-01T00:00:00+00:00")] = [ - object_list_2[0] - ] - ans_buckets_2[datetime.fromisoformat("2022-01-02T00:00:00+00:00")] = [ - object_list_2[1], - object_list_2[2], - ] - ans_buckets_2[datetime.fromisoformat("2022-01-03T00:00:00+00:00")] = [] - assert ans_buckets_2 == generate_expanded_buckets( - object_list_2, Interval(from_time_2, to_time_2), attribute, granularity - ) - - -def test_daily_buckets_without_objects(): - object_list_3 = [] - from_time_3 = datetime(2022, 1, 1, tzinfo=pytz.UTC) - to_time_3 = datetime(2022, 1, 3, tzinfo=pytz.UTC) - attribute = "state_changed_at" - granularity = "daily" - ans_buckets_3 = defaultdict(list) - ans_buckets_3[datetime.fromisoformat("2022-01-01T00:00:00+00:00")] = [] - ans_buckets_3[datetime.fromisoformat("2022-01-02T00:00:00+00:00")] = [] - ans_buckets_3[datetime.fromisoformat("2022-01-03T00:00:00+00:00")] = [] - assert ans_buckets_3 == generate_expanded_buckets( - object_list_3, Interval(from_time_3, to_time_3), attribute, granularity - ) - - -def test_monthly_buckets(): - object_list = [ - AnyObject(datetime(2022, 1, 1, tzinfo=pytz.UTC)), - AnyObject(datetime(2022, 2, 1, tzinfo=pytz.UTC)), - AnyObject(datetime(2022, 2, 15, tzinfo=pytz.UTC)), - AnyObject(datetime(2022, 3, 1, tzinfo=pytz.UTC)), - AnyObject(datetime(2022, 3, 15, tzinfo=pytz.UTC)), - ] - from_time = datetime(2022, 1, 1, tzinfo=pytz.UTC) - to_time = datetime(2022, 3, 15, tzinfo=pytz.UTC) - attribute = "state_changed_at" - granularity = "monthly" - ans_buckets = defaultdict(list) - ans_buckets[datetime.fromisoformat("2022-01-01T00:00:00+00:00")] = [object_list[0]] - ans_buckets[datetime.fromisoformat("2022-02-01T00:00:00+00:00")] = object_list[1:3] - ans_buckets[datetime.fromisoformat("2022-03-01T00:00:00+00:00")] = object_list[3:] - assert ans_buckets == generate_expanded_buckets( - object_list, Interval(from_time, to_time), attribute, granularity - ) - - -def test_data_generates_empty_middle_buckets_for_monthly(): - first_month_2023 = datetime(2023, 1, 2, 0, 0, 0, tzinfo=pytz.UTC) - second_month_2023 = datetime(2023, 2, 9, 0, 0, 0, tzinfo=pytz.UTC) - third_month_2023 = datetime(2023, 3, 16, 0, 0, 0, tzinfo=pytz.UTC) - - from_time = first_month_2023 + timedelta(days=1) - to_time = third_month_2023 + timedelta(days=5) - - obj1 = AnyObject(first_month_2023 + timedelta(days=2)) - obj2 = AnyObject(first_month_2023 + timedelta(days=3)) - obj3 = AnyObject(first_month_2023 + timedelta(days=4)) - obj4 = AnyObject(third_month_2023 + timedelta(days=2)) - obj5 = AnyObject(third_month_2023 + timedelta(days=3)) - obj6 = AnyObject(third_month_2023 + timedelta(days=4)) - object_list = [obj1, obj2, obj3, obj4, obj5, obj6] - - attribute = "state_changed_at" - granularity = "monthly" - - ans_buckets: Dict[datetime, List[Any]] = defaultdict(list) - - ans_buckets[first_month_2023.replace(day=1)] = [obj1, obj2, obj3] - ans_buckets[second_month_2023.replace(day=1)] = [] - ans_buckets[third_month_2023.replace(day=1)] = [obj4, obj5, obj6] - - assert ans_buckets == generate_expanded_buckets( - object_list, Interval(from_time, to_time), attribute, granularity - )