From b4a013b277bb3f67bff5763b7e21ceaba696d35f Mon Sep 17 00:00:00 2001 From: "matt.mercer" Date: Wed, 17 Mar 2021 21:15:34 +0000 Subject: [PATCH] apm-1486: get developer emails and allow secret access to ecs execution role --- .gitignore | 3 +- ansible/Makefile | 5 +- ansible/filter_plugins/apigee_helpers.py | 77 ++++++++++++++++++- ansible/get-developer-emails.yml | 22 ++++++ .../templates/terraform/iam.tf | 34 +++++++- .../templates/terraform/sg.tf | 2 + .../roles/get-developer-emails/tasks/main.yml | 74 ++++++++++++++++++ .../tasks/output-emails.yml | 7 ++ .../roles/get-developer-emails/vars/main.yml | 16 ++++ 9 files changed, 234 insertions(+), 6 deletions(-) create mode 100644 ansible/get-developer-emails.yml create mode 100644 ansible/roles/get-developer-emails/tasks/main.yml create mode 100644 ansible/roles/get-developer-emails/tasks/output-emails.yml create mode 100644 ansible/roles/get-developer-emails/vars/main.yml diff --git a/.gitignore b/.gitignore index b59019d1..b0340eec 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ share/python-wheels/ *.egg MANIFEST .venv/ +*.orig # PyInstaller # Usually these files are written by a python script from a template @@ -148,4 +149,4 @@ env .vscode # ansible testing -ansible/collections/ansible_collections/nhsd/apigee/tests/output/ \ No newline at end of file +ansible/collections/ansible_collections/nhsd/apigee/tests/output/ diff --git a/ansible/Makefile b/ansible/Makefile index 97805db1..c63b405b 100644 --- a/ansible/Makefile +++ b/ansible/Makefile @@ -82,4 +82,7 @@ remove-old-pr-proxies: @poetry run ansible-playbook -i local remove-old-pr-proxies.yml deploy-api-pipelines: guard-PREFIX guard-REPO guard-SERVICE_CONNECTION_ID - PREFIX="$(PREFIX)" ORG="NHSD-APIM" PROJECT="API Platform" REPO="https://github.com/NHSDigital/$(REPO)" SERVICE_CONNECTION="$(SERVICE_CONNECTION_ID)" poetry run ansible-playbook ansible/deploy-api-pipelines.yml \ No newline at end of file + PREFIX="$(PREFIX)" ORG="NHSD-APIM" PROJECT="API Platform" REPO="https://github.com/NHSDigital/$(REPO)" SERVICE_CONNECTION="$(SERVICE_CONNECTION_ID)" poetry run ansible-playbook ansible/deploy-api-pipelines.yml + +get-developer-emails: + @poetry run ansible-playbook -i local get-developer-emails.yml diff --git a/ansible/filter_plugins/apigee_helpers.py b/ansible/filter_plugins/apigee_helpers.py index 773f8dfc..e9115ab4 100644 --- a/ansible/filter_plugins/apigee_helpers.py +++ b/ansible/filter_plugins/apigee_helpers.py @@ -73,6 +73,75 @@ def apigee_remove_proxy_from_product(product: dict, proxy_to_remove): return product +def apigee_teams_map(teams: List[dict]): + + return { + f"{team['id']}@devteam.apigee.io": { + "contact": team['pointOfContact'], + "members": list(mem['userId'] for mem in team.get('memberships', [])) + } + for team in teams + } + + +def apigee_teams_to_point_of_contact(teams: List[dict]): + + return { + f"{team['id']}@devteam.apigee.io": team['pointOfContact'] for team in teams + } + + +def apigee_teams_to_members(teams: List[dict]): + + return { + f"{team['id']}@devteam.apigee.io": list(mem['userId'] for mem in team.get('memberships', [])) for team in teams + } + + +def apigee_product_developers( + product_app_map: dict, dev_id_to_email: dict, teams_map: dict, product_filter: str = None +): + if not product_app_map: + raise ValueError('product_app_map not set') + + if not dev_id_to_email: + raise ValueError('dev_id_to_email not set') + + teams_map = teams_map or {} + result = {} + for product_name, apps in product_app_map.items(): + if product_filter and not re.match(product_filter, product_name): + continue + if product_name not in result: + result[product_name] = [] + + for app in apps: + developer = dev_id_to_email[app["developerId"]] + + team = teams_map.get(developer, {}) + + entry = { + "app": app["appName"], + "developer": developer + } + + if team: + entry['developer'] = team['contact'] + entry['team'] = team['members'] + + result[product_name].append(entry) + + result = { + key: sorted(apps, key=lambda x: x['app']) for key, apps in result.items() + } + + return result + + # return { + # f"{team['id']}@devteam.apigee.io": list(mem['userId'] for mem in team.get('memberships', [])) for team in teams + # } + + class FilterModule: @staticmethod @@ -81,5 +150,9 @@ def filters(): # jinja2 overrides 'apigee_apps_to_product_map': apigee_apps_to_product_map, 'apigee_products_to_api_map': apigee_products_to_api_map, - 'apigee_remove_proxy_from_product': apigee_remove_proxy_from_product - } \ No newline at end of file + 'apigee_remove_proxy_from_product': apigee_remove_proxy_from_product, + 'apigee_teams_to_point_of_contact': apigee_teams_to_point_of_contact, + 'apigee_teams_to_members': apigee_teams_to_members, + 'apigee_teams_map': apigee_teams_map, + 'apigee_product_developers': apigee_product_developers + } diff --git a/ansible/get-developer-emails.yml b/ansible/get-developer-emails.yml new file mode 100644 index 00000000..a6f97c20 --- /dev/null +++ b/ansible/get-developer-emails.yml @@ -0,0 +1,22 @@ +- name: remove old pr portal apis + hosts: 127.0.0.1 + connection: local + gather_facts: yes + + vars: + APIGEE_ORGANIZATION: "{{ lookup('env', 'APIGEE_ORGANIZATION') }}" + APIGEE_ACCESS_TOKEN: "{{ lookup('env', 'APIGEE_ACCESS_TOKEN') }}" + + pre_tasks: + - name: check APIGEE_ORGANIZATION + fail: + msg: "APIGEE_ORGANIZATION not set" + when: not APIGEE_ORGANIZATION + + - name: check APIGEE_ACCESS_TOKEN + fail: + msg: "APIGEE_ACCESS_TOKEN not set" + when: not APIGEE_ACCESS_TOKEN + + roles: + - get-developer-emails diff --git a/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/iam.tf b/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/iam.tf index 0d5cd8ca..007f515b 100644 --- a/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/iam.tf +++ b/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/iam.tf @@ -24,6 +24,29 @@ data "aws_iam_policy_document" "ecs-execution-role" { ] } + statement { + actions = [ + "secretsmanager:GetSecretValue" + ] + + resources = [ + "arn:aws:secretsmanager:${local.region}:${local.account_id}:secret:${var.account}/*" + ] + + condition { + test = "ForAnyValue:StringLike" + values = [ + "${var.service_id}", + "${var.service_id} *", + "* ${var.service_id} *", + "* ${var.service_id}", + "all" + ] + variable = "secretsmanager:ResourceTag/AllowedServices1" + } + + } + statement { actions = [ @@ -66,6 +89,8 @@ resource "aws_iam_role" "ecs-execution-role" { tags = { Name = "ecs-x-${local.env_service_id}" source = "terraform" + api-service = var.service_id + api-environment = var.apigee_environment } } @@ -108,6 +133,12 @@ data "aws_iam_policy_document" "deploy-user-assume-role" { resource "aws_iam_role" "deploy-user" { name = "deploy-${local.env_service_id}" assume_role_policy = data.aws_iam_policy_document.deploy-user-assume-role.json + + tags = { + source = "terraform" + api-service = var.service_id + api-environment = var.apigee_environment + } } resource "aws_iam_role_policy_attachment" "deploy-user" { @@ -202,8 +233,7 @@ data "aws_iam_policy_document" "deploy-user" { condition { test = "StringEquals" - values = [ - var.service_id] + values = [var.service_id] variable = "aws:RequestTag/api-service" } diff --git a/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/sg.tf b/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/sg.tf index 2021b695..5353f141 100644 --- a/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/sg.tf +++ b/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/sg.tf @@ -8,5 +8,7 @@ resource "aws_security_group" "api-deployment" { tags = { Name = "api-${local.env_service_id}" source = "terraform" + api-service = var.service_id + api-environment = var.apigee_environment } } diff --git a/ansible/roles/get-developer-emails/tasks/main.yml b/ansible/roles/get-developer-emails/tasks/main.yml new file mode 100644 index 00000000..bbbfa71b --- /dev/null +++ b/ansible/roles/get-developer-emails/tasks/main.yml @@ -0,0 +1,74 @@ + +- name: "get developers" + uri: + url: "{{ developers_uri }}?expand=true" + headers: + Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}" + return_content: yes + register: get_developers + +- name: "map developers" + set_fact: + dev_id_to_email: "{{ get_developers.json.developer | dict_list_to_map('developerId', 'email') }}" + +- name: get products + uri: + url: "{{ products_uri }}" + headers: + Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}" + return_content: yes + register: products + +- name: list portals + uri: + url: "{{ portals_base_uri }}?orgname={{ APIGEE_ORGANIZATION }}" + headers: + Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}" + return_content: yes + register: portals + +- name: set portal_id + set_fact: + portal_id: "{{ portals.json.data[0].id }}" + +- name: get audience + uri: + url: "https://apigee.com/portals/api/sites/{{ portal_id }}/audiencesenabled" + headers: + Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}" + return_content: yes + register: get_audiences + +- name: set audience_id + set_fact: + audience_id: "{{ get_audiences.json.data.zmsId }}" + +- name: get teams + uri: + url: "https://apigee.com/consumers/api/providers/{{ audience_id }}/teams" + headers: + Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}" + return_content: yes + register: get_teams + +- name: setup developer teams map + set_fact: + teams_map: "{{ get_teams.json.data | apigee_teams_map }}" + +- name: "get apps" + uri: + url: "{{ apps_uri }}?expand=true" + headers: + Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}" + return_content: yes + register: get_apps + +- name: create product map + set_fact: + product_app_map: "{{ get_apps.json.app | apigee_apps_to_product_map(product_filter=product_filter) }}" + +- name: get product developers + set_fact: + product_developers: "{{ product_app_map | apigee_product_developers(dev_id_to_email=dev_id_to_email, teams_map=teams_map) }}" + +- debug: var=product_developers \ No newline at end of file diff --git a/ansible/roles/get-developer-emails/tasks/output-emails.yml b/ansible/roles/get-developer-emails/tasks/output-emails.yml new file mode 100644 index 00000000..b59cf486 --- /dev/null +++ b/ansible/roles/get-developer-emails/tasks/output-emails.yml @@ -0,0 +1,7 @@ + + +- debug: msg="{{ product_slug }} - {{ item.appName}} - {{ dev_id_to_email[item.developerId] }} - {{ apigee_teams_to_members[dev_id_to_email[item.developerId]] | default(dev_id_to_email[item.developerId]) }}" + loop: "{{ product_app_map.get(product_slug, []) }}" + loop_control: + label: "{{ product_slug }} - {{ item.appName }} - {{ item.appId }}" + diff --git a/ansible/roles/get-developer-emails/vars/main.yml b/ansible/roles/get-developer-emails/vars/main.yml new file mode 100644 index 00000000..4ea97547 --- /dev/null +++ b/ansible/roles/get-developer-emails/vars/main.yml @@ -0,0 +1,16 @@ +org_uri: "https://api.enterprise.apigee.com/v1/organizations/{{ APIGEE_ORGANIZATION }}" +products_uri: "{{ org_uri }}/apiproducts" +apps_uri: "{{ org_uri }}/apps" +developers_uri: "{{ org_uri }}/developers" +retain_hours: "{{ (lookup('env', 'retain_hours') or 72) }}" +product_filter: "{{ lookup('env', 'PRODUCT_FILTER') }}" +portals_base_uri: "https://apigee.com/portals/api/sites" + +# https://apigee.com/portals/api/sites/nhsd-nonprod-developerportal/audiencesenabled +# +# nhsd-nonprod/portals/nhsd-nonprod-developerportal/accounts/teams +#https://apigee.com/consumers/api/providers/4ac22135-49bf-4e7e-b79b-93ad2ceb3aa3/teams +# +# https://apigee.com/organizations/nhsd-nonprod/portals/nhsd-nonprod-developerportal/accounts/teams +# https://apigee.com/portals/api/sites/nhsd-nonprod-developerportal/accounts/teams +# "{{ portals_base_uri }}/{{ portal_id }}/apidocs" \ No newline at end of file