From fa9cbfcd0f9425577c42c9b19f741c6eeb31ba48 Mon Sep 17 00:00:00 2001 From: Igor Gentil Date: Mon, 11 Sep 2023 19:51:16 +1000 Subject: [PATCH 1/7] chore: remove unused code and forgotten comments --- unicorn_contracts/tests/unit/conftest.py | 12 ---------- .../tests/unit/event_generator.py | 23 ------------------- .../tests/unit/test_contract_event_handler.py | 2 +- 3 files changed, 1 insertion(+), 36 deletions(-) diff --git a/unicorn_contracts/tests/unit/conftest.py b/unicorn_contracts/tests/unit/conftest.py index e1fa076..e28f236 100644 --- a/unicorn_contracts/tests/unit/conftest.py +++ b/unicorn_contracts/tests/unit/conftest.py @@ -18,18 +18,6 @@ def aws_credentials(): os.environ['AWS_SESSION_TOKEN'] = 'testing' -@pytest.fixture(scope='function') -def env_vars(): - os.environ['POWERTOOLS_SERVICE_NAME']='unicorn.contracts' - os.environ['SERVICE_NAMESPACE']='unicorn.contracts' - os.environ['POWERTOOLS_SERVICE_NAME']='unicorn.contracts' - os.environ['POWERTOOLS_TRACE_DISABLED']='true' - os.environ['POWERTOOLS_LOGGER_LOG_EVENT']='Info' - os.environ['POWERTOOLS_LOGGER_SAMPLE_RATE']='0.1' - os.environ['POWERTOOLS_METRICS_NAMESPACE']='unicorn.contracts' - os.environ['LOG_LEVEL']='INFO' - - @pytest.fixture(scope='function') def dynamodb(aws_credentials): with mock_dynamodb(): diff --git a/unicorn_contracts/tests/unit/event_generator.py b/unicorn_contracts/tests/unit/event_generator.py index 334380c..66eff96 100644 --- a/unicorn_contracts/tests/unit/event_generator.py +++ b/unicorn_contracts/tests/unit/event_generator.py @@ -152,26 +152,3 @@ def sqs_event(messages: List[dict], }) return SQSEvent({ "Records": records }) - - - - -# { -# "Records": [ -# { -# "messageAttributes": { -# "HttpMethod": { -# "stringValue": "PUT", -# "stringListValues": [], -# "binaryListValues": [], -# "dataType": "String" -# } -# }, -# "md5OfMessageAttributes": "39c36267fdf9c8d354b1069e44662d24", -# "md5OfBody": "4bc398b7ae8e52d7f7eaed3cb76c12ef", -# "eventSource": "aws:sqs", -# "eventSourceARN": "arn:aws:sqs:ap-southeast-2:718758479978:uni-prop-local-contract-UnicornContractsIngestQueue-p5zisiK0Xbxn", -# "awsRegion": "ap-southeast-2" -# } -# ] -# } \ No newline at end of file diff --git a/unicorn_contracts/tests/unit/test_contract_event_handler.py b/unicorn_contracts/tests/unit/test_contract_event_handler.py index 3673dcc..25c6440 100644 --- a/unicorn_contracts/tests/unit/test_contract_event_handler.py +++ b/unicorn_contracts/tests/unit/test_contract_event_handler.py @@ -44,7 +44,7 @@ def test_valid_create_event(dynamodb, sqs, lambda_context): def test_valid_update_event(dynamodb, sqs, lambda_context): payload = load_event('update_contract_valid_1') event = sqs_event([{'body': payload, 'attributes': {'HttpMethod': 'PUT'}}]) - + # Loading function here so that mocking works correctly. from contracts_service import contract_event_handler # noqa: F401 # Reload is required to prevent function setup reuse from another test From 35186b3ec38b6a63a9893759b5767e7869a2d26c Mon Sep 17 00:00:00 2001 From: Igor Gentil Date: Mon, 11 Sep 2023 19:52:01 +1000 Subject: [PATCH 2/7] chore: reorder ENV vars; make LambdaContext mock generic to fit all functions --- unicorn_contracts/tests/unit/conftest.py | 6 +++--- unicorn_contracts/tests/unit/helper.py | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/unicorn_contracts/tests/unit/conftest.py b/unicorn_contracts/tests/unit/conftest.py index e28f236..e5ef82d 100644 --- a/unicorn_contracts/tests/unit/conftest.py +++ b/unicorn_contracts/tests/unit/conftest.py @@ -39,12 +39,12 @@ def sqs(aws_credentials): @pytest.fixture(scope='function') def lambda_context(): context: LambdaContext = LambdaContext() - context._function_name="contractsService-CreateContractFunction-IWaQgsTEtLtX" + context._function_name="contractsService-LambdaFunction-IWaQgsTEtLtX" context._function_version="$LATEST" - context._invoked_function_arn="arn:aws:lambda:ap-southeast-2:424490683636:function:contractsService-CreateContractFunction-IWaQgsTEtLtX" + context._invoked_function_arn="arn:aws:lambda:ap-southeast-2:424490683636:function:contractsService-LambdaFunction-IWaQgsTEtLtX" context._memory_limit_in_mb=128 context._aws_request_id="6f970d26-71d6-4c87-a196-9375f85c7b07" - context._log_group_name="/aws/lambda/contractsService-CreateContractFunction-IWaQgsTEtLtX" + context._log_group_name="/aws/lambda/contractsService-LambdaFunction-IWaQgsTEtLtX" context._log_stream_name="2022/07/14/[$LATEST]7c71ca59882b4c569dd007c7e41c81e8" # context._identity=CognitoIdentity([cognito_identity_id=None,cognito_identity_pool_id=None])]) # context._client_context=None diff --git a/unicorn_contracts/tests/unit/helper.py b/unicorn_contracts/tests/unit/helper.py index a1f748a..44a5c8b 100644 --- a/unicorn_contracts/tests/unit/helper.py +++ b/unicorn_contracts/tests/unit/helper.py @@ -20,15 +20,16 @@ def return_env_vars_dict(k=None): env_dict = { "AWS_DEFAULT_REGION": "ap-southeast-2", + "EVENT_BUS": EVENTBUS_NAME, "DYNAMODB_TABLE": TABLE_NAME, - "EVENT_BUS": "test-eventbridge", - "LOG_LEVEL":"INFO", + "SERVICE_NAMESPACE": "unicorn.contracts", + "POWERTOOLS_LOGGER_CASE": "PascalCase", + "POWERTOOLS_SERVICE_NAME":"unicorn.contracts", + "POWERTOOLS_TRACE_DISABLED":"true", "POWERTOOLS_LOGGER_LOG_EVENT":"true", "POWERTOOLS_LOGGER_SAMPLE_RATE":"0.1", "POWERTOOLS_METRICS_NAMESPACE":"unicorn.contracts", - "POWERTOOLS_SERVICE_NAME":"unicorn.contracts", - "POWERTOOLS_TRACE_DISABLED":"true", - "SERVICE_NAMESPACE": "unicorn.contracts", + "LOG_LEVEL":"INFO", } env_dict |= k From ab8aa295a1d781376a1350d6ab446c3e8b914b05 Mon Sep 17 00:00:00 2001 From: Igor Gentil Date: Mon, 11 Sep 2023 19:52:40 +1000 Subject: [PATCH 3/7] chore: lint Python files --- .../properties_service/properties_approval_sync_function.py | 4 ++-- .../properties_service/wait_for_contract_approval_function.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/unicorn_properties/src/properties_service/properties_approval_sync_function.py b/unicorn_properties/src/properties_service/properties_approval_sync_function.py index 6030c85..0de89ab 100644 --- a/unicorn_properties/src/properties_service/properties_approval_sync_function.py +++ b/unicorn_properties/src/properties_service/properties_approval_sync_function.py @@ -28,7 +28,7 @@ @metrics.log_metrics(capture_cold_start_metric=True) # type: ignore -@logger.inject_lambda_context(log_event=True) # type: ignore +@logger.inject_lambda_context(log_event=True) @tracer.capture_method def lambda_handler(event, context): """Functions processes DynamoDB Stream to detect changes in the contract status @@ -76,7 +76,7 @@ def lambda_handler(event, context): return result - +@tracer.capture_method def task_successful(task_token: str, contract_status: dict): """Send the token for a specified contract status back to Step Functions to continue workflow execution. diff --git a/unicorn_properties/src/properties_service/wait_for_contract_approval_function.py b/unicorn_properties/src/properties_service/wait_for_contract_approval_function.py index f945db6..1103fb2 100644 --- a/unicorn_properties/src/properties_service/wait_for_contract_approval_function.py +++ b/unicorn_properties/src/properties_service/wait_for_contract_approval_function.py @@ -31,7 +31,7 @@ @metrics.log_metrics(capture_cold_start_metric=True) # type: ignore -@logger.inject_lambda_context(log_event=True) # type: ignore +@logger.inject_lambda_context(log_event=True) @tracer.capture_method def lambda_handler(event, context): """Function checks to see whether the contract status exists and waits for APPROVAL From 0dac0018597e2e95150676afd30049230b404f22 Mon Sep 17 00:00:00 2001 From: Igor Gentil Date: Mon, 11 Sep 2023 19:53:08 +1000 Subject: [PATCH 4/7] test: add tracer and logger output on propertes_approval_sync_function methods --- .../src/properties_service/properties_approval_sync_function.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unicorn_properties/src/properties_service/properties_approval_sync_function.py b/unicorn_properties/src/properties_service/properties_approval_sync_function.py index 0de89ab..2f4e111 100644 --- a/unicorn_properties/src/properties_service/properties_approval_sync_function.py +++ b/unicorn_properties/src/properties_service/properties_approval_sync_function.py @@ -89,9 +89,11 @@ def task_successful(task_token: str, contract_status: dict): Contract Status object to return to statemachine. """ output = {'Payload': contract_status} + logger.info(output) return sfn.send_task_success(taskToken=task_token, output=json.dumps(output)) +@tracer.capture_method def ddb_deserialize(dynamo_image: dict) -> dict: """Converts the DynamoDB stream object to json dict From 580ab3951e75aa95ea1981345f5e2173839be278 Mon Sep 17 00:00:00 2001 From: Igor Gentil Date: Mon, 11 Sep 2023 19:53:51 +1000 Subject: [PATCH 5/7] chore: reformat StateMachine ASL file --- .../state_machine/property_approval.asl.yaml | 98 +++++++++---------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/unicorn_properties/state_machine/property_approval.asl.yaml b/unicorn_properties/state_machine/property_approval.asl.yaml index 4cdff43..d178f53 100644 --- a/unicorn_properties/state_machine/property_approval.asl.yaml +++ b/unicorn_properties/state_machine/property_approval.asl.yaml @@ -8,23 +8,23 @@ StartAt: VerifyContractExists States: VerifyContractExists: - Type: Task - Resource: arn:aws:states:::lambda:invoke - InputPath: $.detail - ResultPath: $.contract_exists_check - Parameters: - Payload: - Input.$: $ - FunctionName: ${ContractExistsChecker} - Next: CheckImageIntegrity - Catch: - - ErrorEquals: - - ContractStatusNotFoundException - Next: NotFound - Comment: >- - ContractExistsChecker checks to see if a contract for a specified - property exists. - + Type: Task + Resource: arn:aws:states:::lambda:invoke + InputPath: $.detail + ResultPath: $.contract_exists_check + Parameters: + Payload: + Input.$: $ + FunctionName: ${ContractExistsChecker} + Next: CheckImageIntegrity + Catch: + - ErrorEquals: + - ContractStatusNotFoundException + Next: NotFound + Comment: >- + ContractExistsChecker checks to see if a contract for a specified + property exists. + NotFound: Type: Fail @@ -50,7 +50,7 @@ States: unsafe content. ItemsPath: "$.detail.images" ResultPath: "$.imageModerations" - + CheckDescriptionSentiment: Type: Task Parameters: @@ -64,27 +64,27 @@ States: Type: Task Resource: arn:aws:states:::lambda:invoke ResultPath: "$.validation_check" - ResultSelector : + ResultSelector: validation_result.$: "$.Payload.validation_result" Parameters: Payload.$: $ FunctionName: ${ContentIntegrityValidator} Retry: - - ErrorEquals: - - Lambda.ServiceException - - Lambda.AWSLambdaException - - Lambda.SdkClientException - IntervalSeconds: 2 - MaxAttempts: 6 - BackoffRate: 2 + - ErrorEquals: + - Lambda.ServiceException + - Lambda.AWSLambdaException + - Lambda.SdkClientException + IntervalSeconds: 2 + MaxAttempts: 6 + BackoffRate: 2 Next: IsContentSafe IsContentSafe: Type: Choice Choices: - - Variable: $.validation_check.validation_result - StringEquals: PASS - Next: WaitForContractApproval + - Variable: $.validation_check.validation_result + StringEquals: PASS + Next: WaitForContractApproval Default: PublicationEvaluationCompletedDeclined PublicationEvaluationCompletedDeclined: @@ -92,16 +92,16 @@ States: Resource: arn:aws:states:::events:putEvents Parameters: Entries: - - Detail: - property_id.$: "$.detail.property_id" - evaluation_result: "DECLINED" - DetailType: PublicationEvaluationCompleted - EventBusName: ${EventBusName} - Source: ${ServiceName} + - Detail: + property_id.$: "$.detail.property_id" + evaluation_result: "DECLINED" + DetailType: PublicationEvaluationCompleted + EventBusName: ${EventBusName} + Source: ${ServiceName} Next: Declined Declined: Type: Succeed - + WaitForContractApproval: Type: Task Resource: arn:aws:states:::lambda:invoke.waitForTaskToken @@ -113,13 +113,13 @@ States: TaskToken.$: $$.Task.Token FunctionName: ${WaitForContractApproval} Retry: - - ErrorEquals: - - Lambda.ServiceException - - Lambda.AWSLambdaException - - Lambda.SdkClientException - IntervalSeconds: 2 - MaxAttempts: 6 - BackoffRate: 2 + - ErrorEquals: + - Lambda.ServiceException + - Lambda.AWSLambdaException + - Lambda.SdkClientException + IntervalSeconds: 2 + MaxAttempts: 6 + BackoffRate: 2 Next: PublicationEvaluationCompletedApproved Comment: ContractStatusChecker @@ -128,12 +128,12 @@ States: Resource: arn:aws:states:::events:putEvents Parameters: Entries: - - Detail: - property_id.$: "$.detail.property_id" - evaluation_result: "APPROVED" - DetailType: PublicationEvaluationCompleted - EventBusName: ${EventBusName} - Source: ${ServiceName} + - Detail: + property_id.$: "$.detail.property_id" + evaluation_result: "APPROVED" + DetailType: PublicationEvaluationCompleted + EventBusName: ${EventBusName} + Source: ${ServiceName} Next: Approved Approved: Type: Succeed From e3677aa4b6953b54f5fe061433c8e88f2bc4c56f Mon Sep 17 00:00:00 2001 From: Igor Gentil Date: Mon, 11 Sep 2023 19:54:59 +1000 Subject: [PATCH 6/7] fix: update Stage name in shared stack --- unicorn_shared/samconfig.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/unicorn_shared/samconfig.yaml b/unicorn_shared/samconfig.yaml index 1afe5f0..234150d 100644 --- a/unicorn_shared/samconfig.yaml +++ b/unicorn_shared/samconfig.yaml @@ -14,5 +14,4 @@ default: on_failure: ROLLBACK capabilities: CAPABILITY_IAM parameter_overrides: - - "Stage=Local" - + - "Stage=local" From 6e14069ead1b339f69137fd9d908f762f892061a Mon Sep 17 00:00:00 2001 From: Igor Gentil Date: Mon, 11 Sep 2023 19:55:42 +1000 Subject: [PATCH 7/7] chore: renamed stack file for unicorn_shared --- unicorn_shared/samconfig.yaml | 1 - .../{uni-prop-shared.yaml => template.yaml} | 64 +++++++++++-------- 2 files changed, 36 insertions(+), 29 deletions(-) rename unicorn_shared/{uni-prop-shared.yaml => template.yaml} (77%) diff --git a/unicorn_shared/samconfig.yaml b/unicorn_shared/samconfig.yaml index 234150d..b77e6c9 100644 --- a/unicorn_shared/samconfig.yaml +++ b/unicorn_shared/samconfig.yaml @@ -8,7 +8,6 @@ default: resolve_s3: true deploy: parameters: - template_file: uni-prop-shared.yaml confirm_changeset: false fail_on_empty_changeset: false on_failure: ROLLBACK diff --git a/unicorn_shared/uni-prop-shared.yaml b/unicorn_shared/template.yaml similarity index 77% rename from unicorn_shared/uni-prop-shared.yaml rename to unicorn_shared/template.yaml index 75ca694..ef2e1b3 100644 --- a/unicorn_shared/uni-prop-shared.yaml +++ b/unicorn_shared/template.yaml @@ -1,34 +1,22 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 AWSTemplateFormatVersion: "2010-09-09" Transform: - - AWS::LanguageExtensions - AWS::Serverless-2016-10-31 Description: > Base infrastructure that will set up the central event bus and S3 image upload bucket. -#### PARAMETERS Parameters: Stage: Type: String - Default: Local + Default: local AllowedValues: - - Local - - Dev - - Prod + - local + - dev + - prod -#### MAPPINGS - Should be removed when values change to lower case -Mappings: - Stage: - Dev: - Value: dev - Prod: - Value: prod - Local: - Value: local - - -#### GLOBALS Globals: Function: Timeout: 15 @@ -43,29 +31,49 @@ Globals: service: Unicorn Base Infrastructure -#### RESOURCES Resources: - # S3 PROPERTY IMAGES BUCKET - UnicornPropertiesImagesBucket: - Type: AWS::S3::Bucket + #### SSM PARAMETERS + # Service Namespaces + UnicornContractsNamespaceParam: + Type: AWS::SSM::Parameter Properties: - BucketName: !Sub - - uni-prop-${st}-images-${AWS::AccountId} - - st: !FindInMap [Stage, !Ref Stage, Value] + Type: String + Name: !Sub /uni-prop/${Stage}/UnicornContractsNamespace + Value: "unicorn.contracts" + UnicornPropertiesNamespaceParam: + Type: AWS::SSM::Parameter + Properties: + Type: String + Name: !Sub /uni-prop/${Stage}/UnicornPropertiesNamespace + Value: "unicorn.properties" + UnicornWebNamespaceParam: + Type: AWS::SSM::Parameter + Properties: + Type: String + Name: !Sub /uni-prop/${Stage}/UnicornWebNamespace + Value: "unicorn.web" + # Images S3 Bucket UnicornPropertiesImagesBucketParam: Type: AWS::SSM::Parameter Properties: Type: String - Name: !Sub /UniProp/${Stage}/ImagesBucket + Name: !Sub /uni-prop/${Stage}/ImagesBucket Value: !Ref UnicornPropertiesImagesBucket - # IMAGE UPLOAD CUSTOM RESOURCE FUNCTION + + #### S3 PROPERTY IMAGES BUCKET + UnicornPropertiesImagesBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub uni-prop-${Stage}-images-${AWS::AccountId} + + + #### IMAGE UPLOAD CUSTOM RESOURCE FUNCTION ImageUploadFunction: Type: AWS::Serverless::Function Properties: Handler: index.lambda_handler - Runtime: python3.9 Policies: - S3CrudPolicy: BucketName: !Ref UnicornPropertiesImagesBucket