diff --git a/backend/analytics_server/mhq/api/request_utils.py b/backend/analytics_server/mhq/api/request_utils.py index 1dfe2abb..ed947559 100644 --- a/backend/analytics_server/mhq/api/request_utils.py +++ b/backend/analytics_server/mhq/api/request_utils.py @@ -1,11 +1,13 @@ from functools import wraps +from typing import Any, Dict, List from uuid import UUID from flask import request from stringcase import snakecase from voluptuous import Invalid from werkzeug.exceptions import BadRequest -from mhq.store.models.code.workflows import WorkflowFilter +from mhq.service.code.models.org_repo import RawOrgRepo +from mhq.store.models.code import WorkflowFilter, CodeProvider from mhq.service.workflows.workflow_filter import get_workflow_filter_processor @@ -75,3 +77,18 @@ def coerce_workflow_filter(filter_data: str) -> WorkflowFilter: return workflow_filter_processor.create_workflow_filter_from_json_string( filter_data ) + + +def coerce_org_repo(repo: Dict[str, str]) -> RawOrgRepo: + return RawOrgRepo( + provider=CodeProvider(repo.get("provider")), + name=repo.get("name"), + org_name=repo.get("org"), + slug=repo.get("slug"), + idempotency_key=repo.get("idempotency_key"), + default_branch=repo.get("default_branch"), + ) + + +def coerce_org_repos(repos: List[Dict[str, str]]) -> List[RawOrgRepo]: + return [coerce_org_repo(repo) for repo in repos] diff --git a/backend/analytics_server/mhq/api/teams.py b/backend/analytics_server/mhq/api/teams.py index d504af93..35e87392 100644 --- a/backend/analytics_server/mhq/api/teams.py +++ b/backend/analytics_server/mhq/api/teams.py @@ -1,13 +1,14 @@ from flask import Blueprint -from typing import List -from voluptuous import Required, Schema, Optional +from typing import Any, Dict, List +from voluptuous import Required, Schema, Optional, All, Coerce +from mhq.service.code.models.org_repo import RawOrgRepo from mhq.api.resources.code_resouces import adapt_org_repo from mhq.service.code.repository_service import get_repository_service from mhq.api.resources.core_resources import adapt_team from mhq.store.models.core.teams import Team from mhq.service.core.teams import get_team_service -from mhq.api.request_utils import dataschema +from mhq.api.request_utils import coerce_org_repos, dataschema from mhq.service.query_validator import get_query_validator app = Blueprint("teams", __name__) @@ -90,3 +91,21 @@ def fetch_team_repos(team_id: str): team_repos = get_repository_service().get_team_repos(team) return [adapt_org_repo(repo) for repo in team_repos] + + +@app.route("/teams//repos", methods={"PUT"}) +@dataschema( + Schema( + { + Required("repos"): All(list, Coerce(coerce_org_repos)), + } + ), +) +def update_team_repos(team_id: str, repos: List[RawOrgRepo]): + + query_validator = get_query_validator() + team: Team = query_validator.team_validator(team_id) + + team_repos = get_repository_service().update_team_repos(team, repos) + + return [adapt_org_repo(repo) for repo in team_repos] diff --git a/backend/analytics_server/mhq/service/code/models/org_repo.py b/backend/analytics_server/mhq/service/code/models/org_repo.py new file mode 100644 index 00000000..8e1e139c --- /dev/null +++ b/backend/analytics_server/mhq/service/code/models/org_repo.py @@ -0,0 +1,13 @@ +from dataclasses import dataclass + +from mhq.store.models.code.enums import CodeProvider + + +@dataclass +class RawOrgRepo: + provider: CodeProvider + name: str + org_name: str + slug: str + idempotency_key: str + default_branch: str diff --git a/backend/analytics_server/mhq/service/code/repository_service.py b/backend/analytics_server/mhq/service/code/repository_service.py index f702cab8..e2eedbb7 100644 --- a/backend/analytics_server/mhq/service/code/repository_service.py +++ b/backend/analytics_server/mhq/service/code/repository_service.py @@ -1,16 +1,188 @@ from typing import List +from mhq.store.models.incidents.services import TeamIncidentService +from mhq.store.repos.incidents import IncidentsRepoService +from mhq.store.models.incidents import OrgIncidentService, IncidentSource +from mhq.utils.time import time_now +from mhq.utils.string import uuid4_str +from mhq.service.code.models.org_repo import RawOrgRepo from mhq.store.models.code import OrgRepo from mhq.store.models.core import Team from mhq.store.repos.code import CodeRepoService class RepositoryService: - def __init__(self, code_repo_service: CodeRepoService): + def __init__( + self, + code_repo_service: CodeRepoService, + incident_repo_service: IncidentsRepoService, + ): self._code_repo_service = code_repo_service + self._incident_repo_service = incident_repo_service def get_team_repos(self, team: Team) -> List[OrgRepo]: return self._code_repo_service.get_team_repos(team_id=str(team.id)) + def update_team_repos( + self, team: Team, raw_org_repos: List[RawOrgRepo] + ) -> List[OrgRepo]: + + updated_repos = self.update_org_repos(team.org_id, raw_org_repos) + self._code_repo_service.update_team_repos(team, updated_repos) + self.set_unused_repos_as_inactive(team.org_id) + self._update_team_incident_services(team, updated_repos) + + return updated_repos + + def set_unused_repos_as_inactive(self, org_id: str) -> List[OrgRepo]: + + active_repos = self._code_repo_service.get_active_org_repos(org_id) + active_repos_used_across_teams = ( + self._code_repo_service.get_org_repos_used_across_teams(org_id) + ) + + active_team_repo_ids = set( + [str(repo.id) for repo in active_repos_used_across_teams] + ) + + for repo in active_repos: + if str(repo.id) not in active_team_repo_ids: + repo.is_active = False + + return self._code_repo_service.update_org_repos(active_repos) + + def update_org_repos( + self, org_id: str, raw_org_repos: List[RawOrgRepo] + ) -> List[OrgRepo]: + + idempotency_keys = [repo.idempotency_key for repo in raw_org_repos] + + existing_org_repos = self._code_repo_service.get_repos_by_idempotency_keys( + idempotency_keys + ) + + updated_org_repos = [] + idempotency_key_to_repo_map = { + repo.idempotency_key: repo for repo in existing_org_repos + } + + for raw_org_repo in raw_org_repos: + + existing_org_repo = idempotency_key_to_repo_map.get( + raw_org_repo.idempotency_key + ) + if existing_org_repo: + + # ToDo update idempotency key to idempotency_key, provider, org. + if str(existing_org_repo.org_id) != str(org_id): + raise Exception( + f"Data integrity error, matching idempotency key across orgs. Team OrgId: {str(org_id)}. Existing Repo OrgID: {str(existing_org_repo.org_id)}. idempotency_key: {raw_org_repo.idempotency_key}" + ) + + existing_org_repo.is_active = True + existing_org_repo.slug = raw_org_repo.slug + existing_org_repo.name = raw_org_repo.name + + updated_org_repos.append(existing_org_repo) + + else: + updated_org_repos.append( + OrgRepo( + id=uuid4_str(), + org_id=org_id, + name=raw_org_repo.name, + provider=raw_org_repo.provider, + org_name=raw_org_repo.org_name, + idempotency_key=raw_org_repo.idempotency_key, + slug=raw_org_repo.slug, + default_branch=raw_org_repo.default_branch, + ) + ) + + return self._code_repo_service.update_org_repos(updated_org_repos) + + def _update_team_incident_services(self, team: Team, org_repos: List[OrgRepo]): + + incident_services = self._update_org_incident_services(team.org_id, org_repos) + + new_team_services = [] + remove_team_services = [] + + curr_team_services = self._incident_repo_service.get_team_incident_services( + team + ) + + curr_team_services_map = { + str(team_service.service_id): team_service + for team_service in curr_team_services + } + + service_ids = [str(service.id) for service in incident_services] + + for team_service in curr_team_services: + if str(team_service.service_id) not in service_ids: + remove_team_services.append(team_service) + + for service_id in service_ids: + if service_id not in curr_team_services_map: + new_team_services.append( + TeamIncidentService( + id=uuid4_str(), + team_id=team.id, + service_id=service_id, + created_at=time_now(), + updated_at=time_now(), + ) + ) + self._incident_repo_service.add_team_incident_services(new_team_services) + self._incident_repo_service.delete_team_incident_services(remove_team_services) + + def _update_org_incident_services( + self, org_id: str, org_repos: List[OrgRepo] + ) -> List[OrgIncidentService]: + org_repo_ids = [str(org_repo.id) for org_repo in org_repos] + incident_services = self._incident_repo_service.get_org_incident_services( + org_id, IncidentSource.GIT_REPO, org_repo_ids + ) + + key_to_incident_service_map = { + incident_service.key: incident_service + for incident_service in incident_services + } + + updated_incident_services = [] + + for repo in org_repos: + + repo_id = str(repo.id) + incident_service = self._adapt_org_incident_service( + repo, key_to_incident_service_map.get(repo_id) + ) + updated_incident_services.append(incident_service) + + return self._incident_repo_service.update_org_incident_services( + updated_incident_services + ) + + def _adapt_org_incident_service( + self, + org_repo: OrgRepo, + org_incident_service: OrgIncidentService = None, + ) -> OrgIncidentService: + + return OrgIncidentService( + id=org_incident_service.id if org_incident_service else uuid4_str(), + org_id=org_repo.org_id, + provider=org_repo.provider, + name=org_repo.name, + key=str(org_repo.id), + meta={}, + created_at=org_incident_service.created_at + if org_incident_service + else time_now(), + updated_at=time_now(), + source_type=IncidentSource.GIT_REPO, + ) + def get_repository_service(): - return RepositoryService(CodeRepoService()) + return RepositoryService(CodeRepoService(), IncidentsRepoService()) diff --git a/backend/analytics_server/mhq/store/repos/code.py b/backend/analytics_server/mhq/store/repos/code.py index ab03e6d2..3427d72c 100644 --- a/backend/analytics_server/mhq/store/repos/code.py +++ b/backend/analytics_server/mhq/store/repos/code.py @@ -4,6 +4,7 @@ from sqlalchemy import or_ from sqlalchemy.orm import defer +from mhq.store.models.core import Team from mhq.store import db, rollback_on_exc from mhq.store.models.code import ( @@ -37,6 +38,61 @@ def get_active_org_repos(self, org_id: str) -> List[OrgRepo]: def update_org_repos(self, org_repos: List[OrgRepo]): [self._db.session.merge(org_repo) for org_repo in org_repos] self._db.session.commit() + return self.get_repos_by_ids([str(repo.id) for repo in org_repos]) + + @rollback_on_exc + def update_team_repos(self, team: Team, org_repos: List[OrgRepo]): + + existing_team_repos = self._db.session.query(TeamRepos).filter( + TeamRepos.team_id == team.id + ) + + for team_repo in existing_team_repos: + team_repo.is_active = False + + repo_id_to_team_repos_map = { + str(team_repo.org_repo_id): team_repo for team_repo in existing_team_repos + } + + updated_team_repos = [] + for repo in org_repos: + team_repo = repo_id_to_team_repos_map.get(str(repo.id)) + if team_repo: + team_repo.is_active = True + else: + team_repo = TeamRepos( + team_id=team.id, + org_repo_id=str(repo.id), + prod_branches=["^" + repo.default_branch + "$"] + if repo.default_branch + else None, + ) + + updated_team_repos.append(team_repo) + + for team_repo in updated_team_repos: + self._db.session.merge(team_repo) + + self._db.session.commit() + + @rollback_on_exc + def get_org_repos_used_across_teams(self, org_id: str) -> List[OrgRepo]: + """ + Returns a list of all active org repos which are also used in teams. + """ + + return ( + self._db.session.query(OrgRepo) + .join(TeamRepos, TeamRepos.org_repo_id == OrgRepo.id) + .join(Team, TeamRepos.team_id == Team.id) + .filter( + OrgRepo.org_id == org_id, + OrgRepo.is_active.is_(True), + TeamRepos.is_active.is_(True), + Team.is_deleted.is_(False), + ) + .all() + ) @rollback_on_exc def save_pull_requests_data( @@ -247,6 +303,17 @@ def get_repos_by_ids(self, ids: List[str]) -> List[OrgRepo]: return self._db.session.query(OrgRepo).filter(OrgRepo.id.in_(ids)).all() + @rollback_on_exc + def get_repos_by_idempotency_keys( + self, idempotency_keys: List[str] + ) -> List[OrgRepo]: + + return ( + self._db.session.query(OrgRepo) + .filter(OrgRepo.idempotency_key.in_(idempotency_keys)) + .all() + ) + @rollback_on_exc def get_team_repos(self, team_id) -> List[OrgRepo]: team_repos = ( diff --git a/backend/analytics_server/mhq/store/repos/incidents.py b/backend/analytics_server/mhq/store/repos/incidents.py index 15bc8a56..859d48c0 100644 --- a/backend/analytics_server/mhq/store/repos/incidents.py +++ b/backend/analytics_server/mhq/store/repos/incidents.py @@ -1,6 +1,8 @@ from typing import List from sqlalchemy import and_ +from mhq.store.models.core.teams import Team +from mhq.store.models.incidents.enums import IncidentSource from mhq.store import db, rollback_on_exc from mhq.store.models.incidents import ( @@ -23,10 +25,30 @@ def __init__(self): self._db = db @rollback_on_exc - def get_org_incident_services(self, org_id: str) -> List[OrgIncidentService]: + def get_org_incident_services( + self, org_id: str, source_type: IncidentSource = None, keys: List[str] = None + ) -> List[OrgIncidentService]: + + query = self._db.session.query(OrgIncidentService).filter( + OrgIncidentService.org_id == org_id + ) + + if source_type: + query = query.filter(OrgIncidentService.source_type == source_type) + + if keys: + query = query.filter(OrgIncidentService.key.in_(keys)) + + return query.all() + + @rollback_on_exc + def get_org_incident_services_by_ids( + self, ids: List[str] + ) -> List[OrgIncidentService]: + return ( self._db.session.query(OrgIncidentService) - .filter(OrgIncidentService.org_id == org_id) + .filter(OrgIncidentService.id.in_(ids)) .all() ) @@ -37,6 +59,30 @@ def update_org_incident_services(self, incident_services: List[OrgIncidentServic for incident_service in incident_services ] self._db.session.commit() + return self.get_org_incident_services_by_ids( + [incident_service.id for incident_service in incident_services] + ) + + @rollback_on_exc + def get_team_incident_services(self, team: Team) -> List[TeamIncidentService]: + + return ( + self._db.session.query(TeamIncidentService) + .filter(and_(TeamIncidentService.team_id == team.id)) + .all() + ) + + @rollback_on_exc + def add_team_incident_services(self, services: List[TeamIncidentService]): + for service in services: + self._db.session.merge(service) + self._db.session.commit() + + @rollback_on_exc + def delete_team_incident_services(self, services: List[TeamIncidentService]): + for service in services: + self._db.session.delete(service) + self._db.session.commit() @rollback_on_exc def get_incidents_bookmark(