diff --git a/ce-dev/ansible/vars/provision/galaxy-requirements.yml b/ce-dev/ansible/vars/provision/galaxy-requirements.yml index 94dcc9df5..8c33b95ff 100644 --- a/ce-dev/ansible/vars/provision/galaxy-requirements.yml +++ b/ce-dev/ansible/vars/provision/galaxy-requirements.yml @@ -2,8 +2,8 @@ roles: - name: geerlingguy.solr - name: geerlingguy.java - - name: cloudalchemy.process_exporter - - name: cloudalchemy.grafana + - name: prometheus.prometheus.process_exporter + - name: grafana.grafana.grafana collections: - name: community.grafana - name: prometheus.prometheus diff --git a/roles/_meta/aws_region/meta/main.yml b/roles/_meta/aws_region/meta/main.yml index 935fc8939..a11491c0e 100644 --- a/roles/_meta/aws_region/meta/main.yml +++ b/roles/_meta/aws_region/meta/main.yml @@ -8,3 +8,4 @@ dependencies: - role: aws/aws_cloudwatch_log_group - role: aws/aws_backup - role: aws/aws_backup_sns + - role: aws/aws_admin_tools diff --git a/roles/aws/aws_acl/defaults/main.yml b/roles/aws/aws_acl/defaults/main.yml index d74402f89..e71b98ac7 100644 --- a/roles/aws/aws_acl/defaults/main.yml +++ b/roles/aws/aws_acl/defaults/main.yml @@ -6,6 +6,7 @@ aws_acl: region: "us-east-1" tags: "{{ _aws_tags }}" recreate: false # set to true to creating the ACL + default_action: "Allow" # Default action if no rules are triggered, can be Block rules: rate_limit: value: 600 # set to 0 to skip rate limit rule, set to a value to set how many requests to allow in period before blocking diff --git a/roles/aws/aws_acl/tasks/create_acl.yml b/roles/aws/aws_acl/tasks/create_acl.yml index 57ca9b287..26cc6c6f1 100644 --- a/roles/aws/aws_acl/tasks/create_acl.yml +++ b/roles/aws/aws_acl/tasks/create_acl.yml @@ -92,7 +92,7 @@ description: "{{ _acl.description }}" scope: "{{ _acl.scope }}" region: "{{ _acl.region }}" - default_action: Allow # or "Block" + default_action: "{{ _acl.default_action }}" # or "Block" sampled_requests: false cloudwatch_metrics: true # or "false" to disable metrics metric_name: test-metric-name # not sure about this name, since each rule also has it's own metrics name (maybe log group name) diff --git a/roles/aws/aws_admin_tools/defaults/main.yml b/roles/aws/aws_admin_tools/defaults/main.yml new file mode 100644 index 000000000..10c07da2f --- /dev/null +++ b/roles/aws/aws_admin_tools/defaults/main.yml @@ -0,0 +1,18 @@ +aws_admin_tools: + runtime: "python3.12" + timeout: 20 + allowed_ips: + - 192.168.1.1/32 # Ip of server with access to API-s + functions: + - name: "GetForecastedCosts" + type: GET + policies: + - "arn:aws:iam::{{ _acc_id }}:policy/CEBillingPolicy" # Custom policy + - name: "ChangeASGScaling" + type: POST + policies: + - arn:aws:iam::aws:policy/AmazonEC2FullAccess + - name: "GetListOfEC2" + type: GET + policies: + - arn:aws:iam::aws:policy/AmazonEC2FullAccess diff --git a/roles/aws/aws_admin_tools/tasks/create.yml b/roles/aws/aws_admin_tools/tasks/create.yml new file mode 100644 index 000000000..45b72a3ca --- /dev/null +++ b/roles/aws/aws_admin_tools/tasks/create.yml @@ -0,0 +1,74 @@ +- name: Create stage on API gateway. + ansible.builtin.command: >- + aws apigateway create-stage + --rest-api-id "{{ _api_gate.id }}" + --stage-name "prod" + --deployment-id "{{ _main_api_deploy.id }}" + --region "{{ _aws_region }}" + register: _main_api_stage + when: _api_index | length == 0 + +- name: Create resources and set methods on API Gateway. + ansible.builtin.include_tasks: create_methods.yml + loop: "{{ aws_admin_tools.functions }}" + +- name: Obtain all information for a single WAF. + community.aws.wafv2_web_acl_info: + name: "{{ _aws_profile }}_admin_tools" + scope: "REGIONAL" + region: "{{ _aws_region }}" + register: _main_waf + +- name: Get list of API gateway resources. + ansible.builtin.command: >- + aws apigateway get-resources + --region "{{ _aws_region }}" + --rest-api-id "{{ _api_gate.id }}" + register: _api_res_list + +- name: Setting previous command output into variable. + ansible.builtin.set_fact: + _api_res_list: "{{ _api_res_list.stdout | from_json | json_query('items') }}" + +- name: Get index of DelMe resource from API gateway. + ansible.builtin.set_fact: + _api_res_index_list: "{{ lookup('ansible.utils.index_of', _api_res_list, 'eq', '/DelMe', 'path', wantlist=True) }}" + when: _api_index | length == 0 + +- name: Delete the initial resource. + ansible.builtin.command: >- + aws apigateway delete-resource + --rest-api-id "{{ _api_gate.id }}" + --resource-id "{{ _api_res_list[_api_res_index_list[0]].id }}" + --region "{{ _aws_region }}" + when: _api_index | length == 0 + +- name: Deploy API gateway prior to attaching WAF. + ansible.builtin.command: >- + aws apigateway create-deployment + --rest-api-id "{{ _api_gate.id }}" + --stage-name "prod" + --region "{{ _aws_region }}" + +- name: Add API gateway to waf. + community.aws.wafv2_resources: + name: "{{ _aws_profile }}_admin_tools" + scope: REGIONAL + state: present + region: "{{ _aws_region }}" + arn: "arn:aws:apigateway:{{ _aws_region }}::/restapis/{{ _api_gate.id }}/stages/prod" + +- name: Generate unique string. + ansible.builtin.set_fact: + _rand_str: "{{ lookup('community.general.random_string', length=8, special=false, min_lower=2, min_numeric=2, min_upper=2) }}" + +- name: Update Lambda triggers. + ansible.builtin.command: >- + aws lambda add-permission + --function-name "API_{{ item.name }}" + --statement-id "{{ item.name }}_{{ _rand_str }}" + --action "lambda:InvokeFunction" + --principal apigateway.amazonaws.com + --source-arn arn:aws:execute-api:{{ _aws_region }}:{{ _acc_id }}:{{ _api_gate.id }}/*/{{ item.type }}/{{ item.name }} + --region {{ _aws_region }} + loop: "{{ aws_admin_tools.functions }}" diff --git a/roles/aws/aws_admin_tools/tasks/create_methods.yml b/roles/aws/aws_admin_tools/tasks/create_methods.yml new file mode 100644 index 000000000..c10a1c391 --- /dev/null +++ b/roles/aws/aws_admin_tools/tasks/create_methods.yml @@ -0,0 +1,80 @@ +- name: Get resources. + ansible.builtin.command: >- + aws apigateway get-resources + --rest-api-id "{{ _api_gate.id }}" + --region "{{ _aws_region }}" + register: _api_old_resource + +- name: Setting previous command output into variable. + ansible.builtin.set_fact: + _api_old_resource: "{{ _api_old_resource.stdout | from_json }}" + +- name: Find the index of existing resource. + ansible.builtin.set_fact: + _api_old_resource_index: "{{ lookup('ansible.utils.index_of', _api_old_resource['items'], 'eq', '/' + item.name, 'path', wantlist=True) }}" + +- name: Delete resource. + ansible.builtin.command: >- + aws apigateway delete-resource + --rest-api-id "{{ _api_gate.id }}" + --resource-id "{{ _api_old_resource['items'][_api_old_resource_index[0]].id }}" + --region "{{ _aws_region }}" + register: _api_old_resource + when: _api_old_resource_index | length > 0 + +- name: Create resource on API gateway. + ansible.builtin.command: >- + aws apigateway create-resource + --rest-api-id "{{ _api_gate.id }}" + --parent-id "{{ _api_res_list[_api_res_index_list[0]].id }}" + --path-part "{{ item.name }}" + --region "{{ _aws_region }}" + register: _api_resource + +- name: Setting previous command output into variable. + ansible.builtin.set_fact: + _api_resource: "{{ _api_resource.stdout | from_json }}" + +- name: Put method on API gateway + ansible.builtin.command: >- + aws apigateway put-method + --rest-api-id "{{ _api_gate.id }}" + --resource-id "{{ _api_resource.id }}" + --http-method "{{ item.type }}" + --authorization-type "NONE" + --no-api-key-required + --region "{{ _aws_region }}" + +- name: Add Lambda for method. + ansible.builtin.command: >- + aws apigateway put-integration + --rest-api-id "{{ _api_gate.id }}" + --resource-id "{{ _api_resource.id }}" + --http-method "{{ item.type }}" + --type AWS + --content-handling CONVERT_TO_TEXT + --request-templates '{ "application/json": "{\"statusCode\": 200}" }' + --integration-http-method POST + --uri "arn:aws:apigateway:{{ _aws_region }}:lambda:path/2015-03-31/functions/arn:aws:lambda:{{ _aws_region }}:{{ _acc_id }}:function:API_{{ item.name }}/invocations" + --region {{ _aws_region }} + +- name: Add method response. + ansible.builtin.command: >- + aws apigateway put-method-response + --rest-api-id "{{ _api_gate.id }}" + --resource-id "{{ _api_resource.id }}" + --http-method "{{ item.type }}" + --status-code "200" + --response-models '{"application/json":"Empty"}' + --region {{ _aws_region }} + +- name: Add integration response. + ansible.builtin.command: >- + aws apigateway put-integration-response + --rest-api-id "{{ _api_gate.id }}" + --resource-id "{{ _api_resource.id }}" + --http-method "{{ item.type }}" + --status-code "200" + --selection-pattern "" + --content-handling "CONVERT_TO_TEXT" + --region {{ _aws_region }} diff --git a/roles/aws/aws_admin_tools/tasks/create_mock.yml b/roles/aws/aws_admin_tools/tasks/create_mock.yml new file mode 100644 index 000000000..7f4843592 --- /dev/null +++ b/roles/aws/aws_admin_tools/tasks/create_mock.yml @@ -0,0 +1,42 @@ +- name: Create MOCK resource on API gateway. + ansible.builtin.command: >- + aws apigateway create-resource + --rest-api-id "{{ _api_gate.id }}" + --parent-id "{{ _api_res_list[_api_res_index_list[0]].id }}" + --path-part "DelMe" + --region "{{ _aws_region }}" + register: _api_resource + +- name: Setting command output into variable. + ansible.builtin.set_fact: + _api_resource: "{{ _api_resource.stdout | from_json }}" + +- name: Put method on API gateway. + ansible.builtin.command: >- + aws apigateway put-method + --rest-api-id "{{ _api_gate.id }}" + --resource-id "{{ _api_resource.id }}" + --http-method "GET" + --authorization-type "NONE" + --no-api-key-required + --region "{{ _aws_region }}" + +- name: Add mock integration. + ansible.builtin.command: >- + aws apigateway put-integration + --rest-api-id "{{ _api_gate.id }}" + --resource-id "{{ _api_resource.id }}" + --http-method GET + --type MOCK + --region {{ _aws_region }} + +- name: Create initial deployent for API gateway. + ansible.builtin.command: >- + aws apigateway create-deployment + --rest-api-id "{{ _api_gate.id }}" + --region "{{ _aws_region }}" + register: _main_api_deploy + +- name: Setting command output into variable. + ansible.builtin.set_fact: + _main_api_deploy: "{{ _main_api_deploy.stdout | from_json }}" diff --git a/roles/aws/aws_admin_tools/tasks/lambda_functions.yml b/roles/aws/aws_admin_tools/tasks/lambda_functions.yml new file mode 100644 index 000000000..0297d3279 --- /dev/null +++ b/roles/aws/aws_admin_tools/tasks/lambda_functions.yml @@ -0,0 +1,48 @@ +- name: Create S3 bucket for lambda functions. + amazon.aws.s3_bucket: + name: "{{ _aws_profile }}-lambda-api-functions" + region: "{{ _aws_region }}" + state: present + +- name: Check and clean any previous python files. + ansible.builtin.file: + path: "/tmp/{{ item.name }}.py" + state: absent + +- name: Write Lambda functions. + ansible.builtin.template: + src: "API_{{ item.name }}.py.j2" + dest: "/tmp/API_{{ item.name }}.py" + +- name: Create a zip archive of Lambda functions. + community.general.archive: + path: "/tmp/API_{{ item.name }}.py" + dest: "/tmp/API_{{ item.name }}.zip" + format: zip + +- name: Place Lambda functions in S3 bucket. + amazon.aws.s3_object: + bucket: "{{ _aws_profile }}-lambda-api-functions" + object: "lambda-functions/API-{{ item.name }}.zip" + src: "/tmp/API_{{ item.name }}.zip" + mode: put + +- name: Get appropriate IAM role for Lambda. + amazon.aws.iam_role_info: + name: "API_{{ item.name }}" + register: _iam_api_lambda + +- name: Create Lambda functions. + amazon.aws.lambda: + name: "API_{{ item.name }}" + description: "Lambda function for {{ item.name }}" + region: "{{ _aws_region }}" + timeout: "{{ aws_admin_tools.timeout }}" + s3_bucket: "{{ _aws_profile }}-lambda-api-functions" + s3_key: "lambda-functions/API-{{ item.name }}.zip" + state: present + runtime: "{{ aws_admin_tools.runtime }}" + role: "{{ _iam_api_lambda.iam_roles[0].arn }}" + handler: "API_{{ item.name }}.lambda_handler" + tags: + Name: "API_{{ item.name }}" diff --git a/roles/aws/aws_admin_tools/tasks/lambda_iam.yml b/roles/aws/aws_admin_tools/tasks/lambda_iam.yml new file mode 100644 index 000000000..f5ac58341 --- /dev/null +++ b/roles/aws/aws_admin_tools/tasks/lambda_iam.yml @@ -0,0 +1,9 @@ +- name: Attach CloudWatch policy. + ansible.builtin.set_fact: + _policies: "{{ item.policies + ['arn:aws:iam::aws:policy/CloudWatchLogsFullAccess'] }}" + +- name: Create a role and attach policies. + amazon.aws.iam_role: + name: "API_{{ item.name }}" + assume_role_policy_document: "{{ lookup('template', 'trusted_entitites.j2') }}" + managed_policies: "{{ _policies }}" diff --git a/roles/aws/aws_admin_tools/tasks/main.yml b/roles/aws/aws_admin_tools/tasks/main.yml new file mode 100644 index 000000000..21c1f0799 --- /dev/null +++ b/roles/aws/aws_admin_tools/tasks/main.yml @@ -0,0 +1,112 @@ +- name: Get account ID for ARN. + ansible.builtin.command: >- + aws sts get-caller-identity + --query Account + --output text + register: _acc_id + +- name: Setting previous command output into variable. + ansible.builtin.set_fact: + _acc_id: "{{ _acc_id.stdout | from_json }}" + +- name: List all API gateways. + community.aws.api_gateway_info: + region: "{{ _aws_region }}" + register: _api_gate_list + +- name: Find the index of admin tools API. + ansible.builtin.set_fact: + _api_index: "{{ lookup('ansible.utils.index_of', _api_gate_list['rest_apis'], 'eq', _aws_profile + '_admin_tools', 'name', wantlist=True) }}" + +- name: Create API gateway. + ansible.builtin.command: >- + aws apigateway create-rest-api + --region "{{ _aws_region }}" + --name "{{ _aws_profile }}_admin_tools" + --description "API for administration functions made automatically by ansible" + --endpoint-configuration "{\"types\": [\"REGIONAL\"]}" + register: _api_gate + when: _api_index | length == 0 + +- name: Setting previous command output into variable. + ansible.builtin.set_fact: + _api_gate: "{{ _api_gate.stdout | from_json }}" + when: _api_index | length == 0 + +- name: Setting _api_index if API already exists. + ansible.builtin.set_fact: + _api_gate: "{{ _api_gate_list.rest_apis[_api_index[0]] }}" + when: _api_index | length > 0 + +- name: Get list of API gateway resources. + ansible.builtin.command: >- + aws apigateway get-resources + --region "{{ _aws_region }}" + --rest-api-id "{{ _api_gate.id }}" + register: _api_res_list + +- name: Setting previous command output into variable. + ansible.builtin.set_fact: + _api_res_list: "{{ _api_res_list.stdout | from_json | json_query('items') }}" + +- name: Get index of / resource from API gateway. + ansible.builtin.set_fact: + _api_res_index_list: "{{ lookup('ansible.utils.index_of', _api_res_list, 'eq', '/', 'path', wantlist=True) }}" + +- name: Create MOCK deployment. + ansible.builtin.include_tasks: create_mock.yml + when: _api_index | length == 0 + +- name: Get all deployments from API gateway. + ansible.builtin.command: >- + aws apigateway get-deployments + --rest-api-id "{{ _api_gate.id }}" + --region "{{ _aws_region }}" + register: _main_api_deploy + when: _api_index | length > 0 + +- name: Set previous command output into variable. + ansible.builtin.set_fact: + _main_api_deploy_tmp: "{{ _main_api_deploy.stdout | from_json }}" + when: _api_index | length > 0 + +- name: Get last item from deployment list. + ansible.builtin.set_fact: + _main_api_deploy: "{{ _main_api_deploy_tmp['items'] | last }}" + when: _api_index | length > 0 + +- name: Configure Lambda IAM policies. + ansible.builtin.include_tasks: lambda_iam.yml + loop: "{{ aws_admin_tools.functions }}" + +- name: Sleep for 5 seconds for IAM roles. + ansible.builtin.wait_for: + timeout: 5 + +- name: Configure Lambda functions. + ansible.builtin.include_tasks: lambda_functions.yml + loop: "{{ aws_admin_tools.functions }}" + +- name: Create WAF for API Gateway. + ansible.builtin.include_role: + name: aws/aws_acl + vars: + aws_acl: + - name: "{{ _aws_profile }}_admin_tools" + description: "ACL rules for API Gateway" + scope: REGIONAL + region: "{{ _aws_region }}" + tags: "{{ _aws_tags }}" + recreate: true + default_action: "Block" + rules: + ip_sets: + - rule_name: "{{ _aws_profile }}_admin_tools" + set_name: "{{ _aws_profile }}_admin_tools" + description: "List of IPs to allow using API - Ansible managed" + action: allow + priority: 1 + list: "{{ aws_admin_tools.allowed_ips }}" + +- name: Create API Gateway resurces. + ansible.builtin.include_tasks: create.yml diff --git a/roles/aws/aws_admin_tools/templates/API_ChangeASGScaling.py.j2 b/roles/aws/aws_admin_tools/templates/API_ChangeASGScaling.py.j2 new file mode 100644 index 000000000..6bed7668b --- /dev/null +++ b/roles/aws/aws_admin_tools/templates/API_ChangeASGScaling.py.j2 @@ -0,0 +1,39 @@ +import json +import calendar +from datetime import datetime +import boto3 + +costExpl = boto3.client('ce') + +def lambda_handler(event, context): + currDay=datetime.now().day + currMonth=datetime.now().month + print(currMonth) + currYear=datetime.now().year + print(currYear) + lastDay=calendar.monthrange(currYear, currMonth) + + if currMonth < 10: + currMonth = '0' + str(currMonth) + nextDay = currDay + 1 + if currDay < 10: + currDay = '0' + str(currDay) + if nextDay < 10: + nextDay = '0' + str(nextDay) + + startDate=str(currYear) + '-' + str(currMonth) + '-' + str(currDay) + endDate=str(currYear) + '-' + str(currMonth) + '-' + str(nextDay) + + estimatedCost = costExpl.get_cost_forecast( + TimePeriod={ + 'Start': startDate, + 'End': endDate + }, + Granularity='MONTHLY', + Metric='BLENDED_COST' + ) + return { + 'statusCode': 200, + 'Amount': estimatedCost['Total']['Amount'] + ' ' + estimatedCost['Total']['Unit'], + 'Between': estimatedCost['ForecastResultsByTime'][0]['TimePeriod']['Start'] + ' - ' + estimatedCost['ForecastResultsByTime'][0]['TimePeriod']['End'] + } diff --git a/roles/aws/aws_admin_tools/templates/API_GetForecastedCosts.py.j2 b/roles/aws/aws_admin_tools/templates/API_GetForecastedCosts.py.j2 new file mode 100644 index 000000000..6bed7668b --- /dev/null +++ b/roles/aws/aws_admin_tools/templates/API_GetForecastedCosts.py.j2 @@ -0,0 +1,39 @@ +import json +import calendar +from datetime import datetime +import boto3 + +costExpl = boto3.client('ce') + +def lambda_handler(event, context): + currDay=datetime.now().day + currMonth=datetime.now().month + print(currMonth) + currYear=datetime.now().year + print(currYear) + lastDay=calendar.monthrange(currYear, currMonth) + + if currMonth < 10: + currMonth = '0' + str(currMonth) + nextDay = currDay + 1 + if currDay < 10: + currDay = '0' + str(currDay) + if nextDay < 10: + nextDay = '0' + str(nextDay) + + startDate=str(currYear) + '-' + str(currMonth) + '-' + str(currDay) + endDate=str(currYear) + '-' + str(currMonth) + '-' + str(nextDay) + + estimatedCost = costExpl.get_cost_forecast( + TimePeriod={ + 'Start': startDate, + 'End': endDate + }, + Granularity='MONTHLY', + Metric='BLENDED_COST' + ) + return { + 'statusCode': 200, + 'Amount': estimatedCost['Total']['Amount'] + ' ' + estimatedCost['Total']['Unit'], + 'Between': estimatedCost['ForecastResultsByTime'][0]['TimePeriod']['Start'] + ' - ' + estimatedCost['ForecastResultsByTime'][0]['TimePeriod']['End'] + } diff --git a/roles/aws/aws_admin_tools/templates/API_GetListOfEC2.py.j2 b/roles/aws/aws_admin_tools/templates/API_GetListOfEC2.py.j2 new file mode 100644 index 000000000..cc6253de3 --- /dev/null +++ b/roles/aws/aws_admin_tools/templates/API_GetListOfEC2.py.j2 @@ -0,0 +1,49 @@ +import json +import boto3 + +# Defining Clients +ec2_cli = boto3.client("ec2", region_name="{{ _aws_region }}") + +def lambda_handler(event, context): + + print("Gathering instance details.") + ec2_instances=ec2_cli.describe_instances() + + instance_exist = False + Ec2_info_list=[] + + for reservation in ec2_instances["Reservations"]: + for instance in reservation["Instances"]: + pub_ip = "" + priv_ip = "" + inst_name = "" + + if "PublicIpAddress" in instance: + pub_ip = instance['PublicIpAddress'] + else: + pub_ip = "-" + if "PrivateIpAddress" in instance: + priv_ip = instance['PrivateIpAddress'] + else: + priv_ip = "-" + + if "Tags" in instance: + for name in instance['Tags']: + if name['Key'] == 'Name': + inst_name = name['Value'] + else: + inst_name = "-" + + new_dict={ + 'EC2 name': inst_name, + 'State': instance['State'], + 'Public IP': pub_ip, + 'Private IP': priv_ip, + 'Instance type': instance['InstanceType'] + } + Ec2_info_list.append(new_dict) + + return { + 'statusCode': 200, + 'EC2 info': Ec2_info_list + } diff --git a/roles/aws/aws_admin_tools/templates/API_tmp.j2 b/roles/aws/aws_admin_tools/templates/API_tmp.j2 new file mode 100644 index 000000000..83608d358 --- /dev/null +++ b/roles/aws/aws_admin_tools/templates/API_tmp.j2 @@ -0,0 +1,8 @@ +import json + +def lambda_handler(event, context): + + return { + 'statusCode': 200, + 'body': "Yey" + } diff --git a/roles/aws/aws_admin_tools/templates/trusted_entitites.j2 b/roles/aws/aws_admin_tools/templates/trusted_entitites.j2 new file mode 100644 index 000000000..fb84ae9de --- /dev/null +++ b/roles/aws/aws_admin_tools/templates/trusted_entitites.j2 @@ -0,0 +1,12 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/roles/aws/aws_backup_validation/defaults/main.yml b/roles/aws/aws_backup_validation/defaults/main.yml index 2e2632376..26f4613d0 100644 --- a/roles/aws/aws_backup_validation/defaults/main.yml +++ b/roles/aws/aws_backup_validation/defaults/main.yml @@ -1,6 +1,6 @@ --- aws_backup_validation: - s3_bucket: "codeenigma-{{ _aws_profile }}-general-storage-{{ _aws_region }}" + s3_bucket: "ce-{{ _aws_profile }}-lambda-functions" name: "RestoreValidation" description: "Restore validation is running every Sunday at 00:00AM, and validation reporting is triggered on Monday 00:00AM" timeout: 60 diff --git a/roles/aws/aws_backup_validation/tasks/main.yml b/roles/aws/aws_backup_validation/tasks/main.yml index 5ebb5d90b..37366f4b2 100644 --- a/roles/aws/aws_backup_validation/tasks/main.yml +++ b/roles/aws/aws_backup_validation/tasks/main.yml @@ -1,162 +1,102 @@ --- -- name: Create a role and attach policies - amazon.aws.iam_role: - name: LambdaBackupRestoreRole - assume_role_policy_document: "{{ lookup('file', 'trusted_entitites.j2') }}" - managed_policies: - - arn:aws:iam::aws:policy/AmazonEC2FullAccess - - arn:aws:iam::aws:policy/AWSBackupFullAccess - - arn:aws:iam::aws:policy/AmazonRDSFullAccess - - arn:aws:iam::aws:policy/CloudWatchLogsFullAccess - - arn:aws:iam::aws:policy/AmazonSESFullAccess - - arn:aws:iam::aws:policy/AmazonSSMFullAccess - register: _created_iam_lambda_role - -- name: Create an IAM Managed Policy for passing roles - amazon.aws.iam_managed_policy: - policy_name: "PassRole" - policy: - Version: "2012-10-17" - Statement: - - Effect: "Allow" - Action: "iam:PassRole" - Resource: "*" - state: present - register: _pass_role - -- name: Update AWSBackupDefaultServiceRole - amazon.aws.iam_role: - name: AWSBackupDefaultServiceRole - assume_role_policy_document: "{{ lookup('file', 'pass_role_backup.j2') }}" - managed_policies: - - arn:aws:iam::aws:policy/service-role/AWSBackupServiceRolePolicyForBackup - - arn:aws:iam::aws:policy/service-role/AWSBackupServiceRolePolicyForRestores - - "{{ _pass_role.policy.arn }}" +- name: Create a role and attach policies for Lambda backup validation. + ansible.builtin.include_role: + name: aws/aws_iam_role + vars: + aws_iam_role: + name: LambdaBackupRestoreRole + aws_profile: "{{ _aws_profile }}" + managed_policies: + - arn:aws:iam::aws:policy/AmazonEC2FullAccess + - arn:aws:iam::aws:policy/AWSBackupFullAccess + - arn:aws:iam::aws:policy/AmazonRDSFullAccess + - arn:aws:iam::aws:policy/CloudWatchLogsFullAccess + - arn:aws:iam::aws:policy/AmazonSSMFullAccess + policy_document: "{{ lookup('file', 'trusted_entitites.j2') }}" + +- name: Create backup validation Lambda functions. + ansible.builtin.include_role: + name: aws/aws_lambda + vars: + aws_lambda: + name: "{{ aws_backup_validation.name }}_{{ item }}" + description: "{{ aws_backup_validation.description }}" + timeout: "{{ aws_backup_validation.timeout }}" + role: "{{ aws_iam_role._result['LambdaBackupRestoreRole'] }}" + runtime: "{{ aws_backup_validation.runtime }}" + function_file: "{{ lookup('template', item + '_validation.py.j2') }}" + s3_bucket: "ce-{{ _aws_profile }}-lambda-functions" + tags: + Name: "{{ item }}_backup_validation" + loop: "{{ aws_backup_validation.resources }}" -- name: Sleep for 10 seconds for IAM before Lambda creation - ansible.builtin.wait_for: - timeout: 10 +#- name: Remove variables containing "-". +# ansible.builtin.set_fact: +# aws_lambda: "{{ aws_lambda | ansible.utils.remove_keys(target=['response_metadata', 'function_file']) }}" + +- name: Create an IAM Managed Policy for passing roles and setup IAM role. + ansible.builtin.include_role: + name: aws/aws_iam_role + vars: + aws_iam_role: + name: AWSBackupDefaultServiceRole + aws_profile: "{{ _aws_profile }}" + inline_policies: + name: "PassRole" + resource: "*" + action: "iam:PassRole" + policy_document: "{{ lookup('file', 'pass_role_backup.j2') }}" + managed_policies: + - arn:aws:iam::aws:policy/service-role/AWSBackupServiceRolePolicyForBackup + - arn:aws:iam::aws:policy/service-role/AWSBackupServiceRolePolicyForRestores # TODO: Not all clients have verified identity #- name: Get verified domain. # ansible.builtin.include_tasks: get_valid_email.yml -- name: Clean and set python functions - block: - - name: Create S3 bucket for lambda functions - amazon.aws.s3_bucket: - name: "{{ aws_backup_validation.s3_bucket }}" - region: "{{ _aws_region }}" - state: present - - - name: Check and clean any previous backup validation files - ansible.builtin.file: - path: "{{ _ce_provision_build_dir }}/{{ item }}_validation.py" - state: absent - loop: "{{ aws_backup_validation.resources }}" - - - name: Check and clean any previous validation report files - ansible.builtin.file: - path: "{{ _ce_provision_build_dir }}/validation_report.py" - state: absent - - - name: Write Lambda functions - ansible.builtin.template: - src: "{{ item }}_validation.py.j2" - dest: "{{ _ce_provision_build_dir }}/{{ item }}_validation.py" - loop: "{{ aws_backup_validation.resources }}" - - - name: Get info about newly created restore testing plan. - ansible.builtin.command: > - aws backup list-restore-testing-plans --region {{ _aws_region }} - register: _testing_plans - - - name: Print return information from the previous task - ansible.builtin.debug: - var: _testing_plans - - - name: Write validation report functions - ansible.builtin.template: - src: "validation_report.j2" - dest: "{{ _ce_provision_build_dir }}/validation_report.py" - - - name: Create a zip archive of Lambda functions - community.general.archive: - path: "{{ _ce_provision_build_dir }}/{{ item }}_validation.py" - dest: "{{ _ce_provision_build_dir }}/{{ item }}_validation.zip" - format: zip - loop: "{{ aws_backup_validation.resources }}" - - - name: Create a zip archive of validation report - community.general.archive: - path: "{{ _ce_provision_build_dir }}/validation_report.py" - dest: "{{ _ce_provision_build_dir }}/validation_report.zip" - format: zip - - - name: Place backup validation functions in S3 bucket - amazon.aws.s3_object: - bucket: "{{ aws_backup_validation.s3_bucket }}" - object: "lambda-functions/{{ item }}_validation.zip" - src: "{{ _ce_provision_build_dir }}/{{ item }}_validation.zip" - mode: put - loop: "{{ aws_backup_validation.resources }}" - - - name: Place report function in S3 bucket - amazon.aws.s3_object: - bucket: "{{ aws_backup_validation.s3_bucket }}" - object: "lambda-functions/validation_report.zip" - src: "{{ _ce_provision_build_dir }}/validation_report.zip" - mode: put - loop: "{{ aws_backup_validation.resources }}" - -- name: Create Lambda functions - amazon.aws.lambda: - name: "{{ aws_backup_validation.name }}_{{ item }}" - description: "{{ aws_backup_validation.description }}" - region: "{{ _aws_region }}" - timeout: "{{ aws_backup_validation.timeout }}" - s3_bucket: "{{ aws_backup_validation.s3_bucket }}" - s3_key: "lambda-functions/{{ item }}_validation.zip" - state: present - runtime: "{{ aws_backup_validation.runtime }}" - role: "{{ _created_iam_lambda_role.iam_role.arn }}" - handler: "{{ item }}_validation.{{ aws_backup_validation.handler }}" - tags: - Name: "{{ item }}_backup_validation" - register: _lambda_functions - loop: "{{ aws_backup_validation.resources }}" - -- name: Create validation report functions - amazon.aws.lambda: - name: "validation_report" - description: "{{ aws_backup_validation.description }}" - region: "{{ _aws_region }}" - timeout: 30 - s3_bucket: "{{ aws_backup_validation.s3_bucket }}" - s3_key: "lambda-functions/validation_report.zip" - state: present - runtime: "{{ aws_backup_validation.runtime }}" - role: "{{ _created_iam_lambda_role.iam_role.arn }}" - handler: "validation_report.{{ aws_backup_validation.handler }}" - register: _validation_report - -- name: Remove non UTF-8 item +- name: Get info about newly created restore testing plan. + ansible.builtin.command: > + aws backup list-restore-testing-plans --region {{ _aws_region }} + register: _testing_plans + +- name: Create validation report function. + ansible.builtin.include_role: + name: aws/aws_lambda + vars: + aws_lambda: + name: "validation_report" + description: "{{ aws_backup_validation.description }}" + timeout: "30" + role: "{{ aws_iam_role._result['LambdaBackupRestoreRole'] }}" + runtime: "{{ aws_backup_validation.runtime }}" + function_file: "{{ lookup('template', 'validation_report.py.j2') }}" + s3_bucket: "ce-{{ _aws_profile }}-lambda-functions" + tags: + Name: "validation_report" + +- name: Get account ID for ARN. + ansible.builtin.command: >- + aws sts get-caller-identity + --query Account + --output text + register: _acc_id + +- name: Setting previous command output into variable. ansible.builtin.set_fact: - _lambda_functions: "{{ _lambda_functions | ansible.utils.remove_keys(target=['ZipFile', 'location', 'item.invocation']) }}" - _validation_report: "{{ _validation_report | ansible.utils.remove_keys(target=['ZipFile', 'location', 'item.invocation']) }}" + _acc_id: "{{ _acc_id.stdout | from_json }}" -- name: Create EventBridge for validations +- name: Create EventBridge for validation functions. amazon.aws.cloudwatchevent_rule: - name: "{{ item.configuration.function_name }}" - description: "{{ item.configuration.description }}" + name: "RestoreValidation_{{ item }}" + description: "{{ aws_backup_validation.description }}" state: present region: "{{ _aws_region }}" - event_pattern: '{ "source": ["aws.backup"], "detail-type": ["Restore Job State Change"], "detail": { "resourceType": ["{{ item.item }}"], "status": ["COMPLETED"] } }' + event_pattern: '{ "source": ["aws.backup"], "detail-type": ["Restore Job State Change"], "detail": { "resourceType": ["{{ item }}"], "status": ["COMPLETED"] } }' targets: - - id: "{{ item.configuration.function_name }}" - arn: "{{ (item.configuration.function_arn.split(':') | map('trim'))[:-1] | join(':') }}" # Remove the version number from ARN + - id: "RestoreValidation_{{ item }}" + arn: "arn:aws:lambda:{{ _aws_region }}:{{ _acc_id }}:function:RestoreValidation_{{ item }}" + loop: "{{ aws_backup_validation.resources }}" register: _event_bridges - loop: "{{ _lambda_functions.results }}" - name: Create schedule for validation reports amazon.aws.cloudwatchevent_rule: @@ -166,7 +106,7 @@ region: "{{ _aws_region }}" targets: - id: validation_report - arn: "{{ (_validation_report.configuration.function_arn.split(':') | map('trim'))[:-1] | join(':') }}" # Remove the version number from ARN + arn: "{{ (aws_lambda._result['validation_report'].configuration.function_arn.split(':') | map('trim'))[:-1] | join(':') }}" # Remove the version number from ARN register: _validation_event - name: Generate unique string @@ -176,8 +116,8 @@ - name: Update Lambda policy amazon.aws.lambda_policy: state: present - function_name: "{{ item.item.configuration.function_name }}" - statement_id: "{{ item.item.configuration.function_name }}_{{ _rand_str }}" + function_name: "{{ item.rule.name }}" + statement_id: "{{ item.rule.name }}_{{ _rand_str }}" action: lambda:InvokeFunction principal: events.amazonaws.com source_arn: "{{ item.rule.arn }}" @@ -188,7 +128,7 @@ amazon.aws.lambda_policy: state: present function_name: "validation_report" - statement_id: "{{ _validation_report.configuration.function_name }}_{{ _rand_str }}" + statement_id: "validation_report_{{ _rand_str }}" action: lambda:InvokeFunction principal: events.amazonaws.com source_arn: "{{ _validation_event.rule.arn }}" diff --git a/roles/aws/aws_backup_validation/templates/validation_report.j2 b/roles/aws/aws_backup_validation/templates/validation_report.py.j2 similarity index 98% rename from roles/aws/aws_backup_validation/templates/validation_report.j2 rename to roles/aws/aws_backup_validation/templates/validation_report.py.j2 index 9c29baea4..bc80f32a7 100644 --- a/roles/aws/aws_backup_validation/templates/validation_report.j2 +++ b/roles/aws/aws_backup_validation/templates/validation_report.py.j2 @@ -98,9 +98,9 @@ failed_job = backup_cli.list_restore_jobs( {% endfor %} if len(failed_jobs) > 0: - mail_title = "Failed!" + mail_title = "🔴 Failed!" else: - mail_title = "Success!" + mail_title = "🟢 Success!" print("Successful restore jobs:") print(completed_jobs) diff --git a/roles/aws/aws_ec2_with_eip/tasks/main.yml b/roles/aws/aws_ec2_with_eip/tasks/main.yml index fea2c4ecb..dee50043d 100644 --- a/roles/aws/aws_ec2_with_eip/tasks/main.yml +++ b/roles/aws/aws_ec2_with_eip/tasks/main.yml @@ -163,23 +163,12 @@ tag:Name: "{{ aws_ec2_with_eip.instance_name }}" register: _aws_ec2_with_eip_instances_eip -- name: Generate Terraform template. - ansible.builtin.template: - src: eip.tf.j2 - dest: "{{ _ce_provision_build_tmp_dir }}/main.tf" - mode: "0666" - when: not _aws_ec2_with_eip_instances_eip.addresses - -- name: Init Terraform. - ansible.builtin.command: - cmd: terraform init - chdir: "{{ _ce_provision_build_tmp_dir }}" - when: not _aws_ec2_with_eip_instances_eip.addresses - -- name: Create EIP with Terraform if we don't have one. - ansible.builtin.command: - cmd: terraform apply -auto-approve - chdir: "{{ _ce_provision_build_tmp_dir }}" +- name: Allocate a new elastic IP inside a VPC. + amazon.aws.ec2_eip: + region: "{{ aws_ec2_with_eip.region }}" + in_vpc: true + tag_name: "Name" + tag_value: "{{ aws_ec2_with_eip.instance_name }}" when: not _aws_ec2_with_eip_instances_eip.addresses - name: Re-register EIP. diff --git a/roles/aws/aws_iam_role/defaults/main.yml b/roles/aws/aws_iam_role/defaults/main.yml index 585728d80..c4bc28ad6 100644 --- a/roles/aws/aws_iam_role/defaults/main.yml +++ b/roles/aws/aws_iam_role/defaults/main.yml @@ -3,6 +3,10 @@ aws_iam_role: aws_profile: "{{ _aws_profile }}" # Pass either names or ARNs for the role. managed_policies: [] + inline_policies: + name: "example_inline_polcy" # Name of inline policy + resource: "*" + action: [] # Which document policy to apply. # Current options are 'ec2', 'ecs' or 'backup' policy_document: ec2 diff --git a/roles/aws/aws_iam_role/tasks/main.yml b/roles/aws/aws_iam_role/tasks/main.yml index daf1ad759..fb6aa3bb0 100644 --- a/roles/aws/aws_iam_role/tasks/main.yml +++ b/roles/aws/aws_iam_role/tasks/main.yml @@ -1,9 +1,42 @@ +- name: Create an IAM Managed Policy if defined. + amazon.aws.iam_managed_policy: + policy_name: "inline_{{ aws_iam_role.name }}_policy" + policy: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: "{{ aws_iam_role.inline_policies.action }}" + Resource: "{{ aws_iam_role.inline_policies.resource }}" + state: present + register: _inline_iam_policy + when: inline_policies.action is defined and inline_policies.action > 0 + +- name: Join managed and inline policy. + ansible.builtin.set_fact: + _combined_policies: "{{ aws_iam_role.managed_policies + [_inline_iam_policy.arn] }}" + when: inline_policies.action is defined and inline_policies.action > 0 + +- name: Create combined var if inline policy is not defined or empty. + ansible.builtin.set_fact: + _combined_policies: "{{ aws_iam_role.managed_policies }}" + when: inline_policies.action is not defined or inline_policies.action == 0 + +- name: Create assume role policy document if predefined string is passed. + ansible.builtin.set_fact: + _assume_role_policy: "{{ lookup('file', aws_iam_role.policy_document + '_document_policy.json') }}" + when: aws_iam_role.policy_document | type_debug == 'str' + +- name: Create assume role policy document if template is provided. + ansible.builtin.set_fact: + _assume_role_policy: "{{ aws_iam_role.policy_document }}" + when: aws_iam_role.policy_document | type_debug != 'str' + - name: Create an IAM role. amazon.aws.iam_role: profile: "{{ aws_iam_role.aws_profile }}" name: "{{ aws_iam_role.name }}" - assume_role_policy_document: "{{ lookup('file', aws_iam_role.policy_document + '_document_policy.json') }}" - managed_policies: "{{ aws_iam_role.managed_policies }}" + assume_role_policy_document: "{{ _assume_role_policy }}" + managed_policies: "{{ _combined_policies }}" purge_policies: "{{ aws_iam_role.purge_policies }}" tags: "{{ aws_iam_role.tags }}" create_instance_profile: "{% if aws_iam_role.policy_document == 'ec2' %}true{% else %}false{% endif %}" @@ -12,4 +45,4 @@ - name: Register aws_iam_role results. ansible.builtin.set_fact: - aws_iam_role: "{{ aws_iam_role | combine({'_result': {aws_iam_role.name: _aws_iam_role_result}}) }}" + aws_iam_role: "{{ aws_iam_role | combine({'_result': {aws_iam_role.name: _aws_iam_role_result}}, recursive=True) }}" diff --git a/roles/aws/aws_lambda/defaults/main.yml b/roles/aws/aws_lambda/defaults/main.yml new file mode 100644 index 000000000..6bcf10abf --- /dev/null +++ b/roles/aws/aws_lambda/defaults/main.yml @@ -0,0 +1,10 @@ +aws_lambda: + name: "lambda_function_name" + description: "Description for AWS Lambda function" + timeout: "20" # Maximum number of seconds before function times out + handler: "lambda_handler" # Name of main function + s3_bucket: "ce-{{ _aws_profile }}-lambda-functions" + function_file: "" # template to pass in S3 bucket + runtime: "python3.12" + role: "" + tags: [] diff --git a/roles/aws/aws_lambda/tasks/main.yml b/roles/aws/aws_lambda/tasks/main.yml new file mode 100644 index 000000000..27b692c50 --- /dev/null +++ b/roles/aws/aws_lambda/tasks/main.yml @@ -0,0 +1,52 @@ +- name: Create S3 bucket for Lambda functions. + ansible.builtin.include_role: + name: aws/aws_s3_bucket + vars: + aws_s3_bucket: + profile: "{{ _aws_profile }}" + region: "{{ _aws_region }}" + name: "{{ aws_lambda.s3_bucket }}" + tags: [] + state: "present" + +- name: Check and clean previous Lambda function. + ansible.builtin.file: + path: "{{ _ce_provision_build_dir }}/{{ aws_lambda.name }}.py" + state: absent + +- name: Write Lambda function. + ansible.builtin.copy: + content: "{{ aws_lambda.function_file }}" + dest: "{{ _ce_provision_build_dir }}/{{ aws_lambda.name }}.py" + +- name: Create a zip archive of Lambda function. + community.general.archive: + path: "{{ _ce_provision_build_dir }}/{{ aws_lambda.name }}.py" + dest: "{{ _ce_provision_build_dir }}/{{ aws_lambda.name }}.zip" + format: zip + +- name: Place Lambda function in S3 bucket. + amazon.aws.s3_object: + bucket: "{{ aws_lambda.s3_bucket }}" + object: "{{ aws_lambda.name }}.zip" + src: "{{ _ce_provision_build_dir }}/{{ aws_lambda.name }}.zip" + mode: put + +- name: Create Lambda function. + amazon.aws.lambda: + name: "{{ aws_lambda.name }}" + description: "{{ aws_lambda.description }}" + region: "{{ _aws_region }}" + timeout: "{{ aws_lambda.timeout }}" + s3_bucket: "{{ aws_lambda.s3_bucket }}" + s3_key: "{{ aws_lambda.name }}.zip" + state: present + runtime: "{{ aws_lambda.runtime }}" + role: "{{ aws_lambda.role.iam_role.arn }}" + handler: "{{ aws_lambda.name }}.{{ aws_lambda.handler }}" + tags: "{{ aws_lambda.tags }}" + register: _aws_lambda_result + +- name: Register aws_lambda results. + ansible.builtin.set_fact: + aws_lambda: "{{ aws_lambda | combine({'_result': {aws_lambda.name: _aws_lambda_result}}, recursive=True) }}" diff --git a/roles/aws/aws_s3_bucket/tasks/main.yml b/roles/aws/aws_s3_bucket/tasks/main.yml index 8abcffa76..915b9c09e 100644 --- a/roles/aws/aws_s3_bucket/tasks/main.yml +++ b/roles/aws/aws_s3_bucket/tasks/main.yml @@ -1,15 +1,15 @@ - name: Create an S3 bucket. amazon.aws.s3_bucket: - profile: "{{ aws_s3_bucket.aws_profile }}" + profile: "{{ _aws_profile }}" region: "{{ aws_s3_bucket.region }}" name: "{{ aws_s3_bucket.name }}" tags: "{{ aws_s3_bucket.tags | combine({'Name': aws_s3_bucket.name}) }}" state: present - register: _aws_s3_bucket_bucket + register: _aws_s3_bucket - name: Create a matching policy. amazon.aws.iam_managed_policy: - profile: "{{ aws_s3_bucket.aws_profile }}" + profile: "{{ _aws_profile }}" region: "{{ aws_s3_bucket.region }}" policy_name: "{{ aws_s3_bucket.name }}" policy: @@ -20,8 +20,8 @@ Resource: "arn:aws:s3:::{{ aws_s3_bucket.name }}" make_default: true state: present - register: _aws_s3_bucket_bucket_policy + register: _aws_s3_bucket_policy - name: Register aws_s3_bucket results. ansible.builtin.set_fact: - aws_s3_bucket: "{{ aws_s3_bucket | combine({'_result': {aws_s3_bucket.name: {'bucket': _aws_s3_bucket_bucket, 'policy': _aws_s3_bucket_bucket_policy}}}) }}" + aws_s3_bucket: "{{ aws_s3_bucket | combine({'_result': {aws_s3_bucket.name | replace('-','_'): {'bucket': _aws_s3_bucket, 'policy': _aws_s3_bucket_policy}}}, recursive=True) }}" diff --git a/roles/aws/aws_sg_iptables/tasks/main.yml b/roles/aws/aws_sg_iptables/tasks/main.yml index baa4fc575..826a29494 100644 --- a/roles/aws/aws_sg_iptables/tasks/main.yml +++ b/roles/aws/aws_sg_iptables/tasks/main.yml @@ -7,7 +7,7 @@ - name: Set list of commands for tcp/udp in include_tasks: tcp_udp_string.yml - loop: "{{ q( 'ansible.builtin.subelements', _glob, 'ports', { 'skip_missing': True }) }}" + loop: "{{ q('ansible.builtin.subelements', _glob, 'ports', {'skip_missing': True}) }}" - name: Remove priority element from dict. ansible.builtin.set_fact: @@ -29,7 +29,7 @@ - name: Set list of commands four tcp/udp out include_tasks: tcp_udp_string.yml - loop: "{{ q( 'ansible.builtin.subelements', _glob, 'ports', { 'skip_missing': True }) }}" + loop: "{{ q('ansible.builtin.subelements', _glob, 'ports', {'skip_missing': True}) }}" - name: Set list of commands for icmp in include_tasks: icmp_string.yml