ci: Make transform test error json easier to diff/review (aws#2968)

Makefile

@@ -19,6 +19,7 @@ integ-test:
 
 black:
 	black setup.py samtranslator tests integration bin schema_source
+	bin/transform-test-error-json-format.py --write tests/translator/output/error_*.json
 	bin/json-format.py --write tests integration samtranslator/policy_templates_data
 	bin/yaml-format.py --write tests
 	bin/yaml-format.py --write integration --add-test-metadata
@@ -30,6 +31,7 @@ black-check:
 	diff -u schema_source/sam.schema.json .tmp/sam.schema.json
 	diff -u samtranslator/schema/schema.json .tmp/schema.json
 	black --check setup.py samtranslator tests integration bin schema_source
+	bin/transform-test-error-json-format.py --check tests/translator/output/error_*.json
 	bin/json-format.py --check tests integration samtranslator/policy_templates_data
 	bin/yaml-format.py --check tests
 	bin/yaml-format.py --check integration --add-test-metadata

bin/transform-test-error-json-format.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""
+Transform test error JSON file formatter (without prettier).
+
+It makes error json easier to review by breaking down "errorMessage"
+into list of strings (delimiter: ". ").
+"""
+import os
+import sys
+
+from typing_extensions import Final
+
+my_path = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, my_path + "/..")
+
+import json
+from typing import Type
+
+from bin._file_formatter import FileFormatter
+
+
+class TransformTestErrorJSONFormatter(FileFormatter):
+    _ERROR_MESSAGE_KEY: Final[str] = "errorMessage"
+    _BREAKDOWN_ERROR_MESSAGE_KEY: Final[str] = "_autoGeneratedBreakdownErrorMessage"
+    _DELIMITER: Final[str] = ". "
+
+    @staticmethod
+    def description() -> str:
+        return "Transform test error JSON file formatter"
+
+    def format_str(self, input_str: str) -> str:
+        """
+        It makes error json easier to review by breaking down "errorMessage"
+        into list of strings (delimiter: ". ").
+        """
+        obj = json.loads(input_str)
+        error_message = obj.get(self._ERROR_MESSAGE_KEY)
+        if isinstance(error_message, str):
+            tokens = error_message.split(self._DELIMITER)
+            obj[self._BREAKDOWN_ERROR_MESSAGE_KEY] = [
+                token if index == len(tokens) - 1 else token + self._DELIMITER for index, token in enumerate(tokens)
+            ]
+        return json.dumps(obj, indent=2, sort_keys=True) + "\n"
+
+    @staticmethod
+    def decode_exception() -> Type[Exception]:
+        return json.JSONDecodeError
+
+    @staticmethod
+    def file_extension() -> str:
+        return ".json"
+
+
+if __name__ == "__main__":
+    TransformTestErrorJSONFormatter.main()

tests/translator/output/error_api_auth_invalid_path_item.json

@@ -1,3 +1,9 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 1. ",
+    "Structure of the SAM template is invalid. ",
+    "Value of '/' path must be a dictionary according to Swagger spec."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Structure of the SAM template is invalid. Value of '/' path must be a dictionary according to Swagger spec."
 }

tests/translator/output/error_api_auth_null_path_item.json

@@ -1,3 +1,9 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 1. ",
+    "Structure of the SAM template is invalid. ",
+    "Value of '/' path must be a dictionary according to Swagger spec."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Structure of the SAM template is invalid. Value of '/' path must be a dictionary according to Swagger spec."
 }

tests/translator/output/error_api_authorizer_property_indentity_header_with_invalid_type.json

@@ -1,3 +1,11 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 2. ",
+    "Resource with id [MyApi] is invalid. ",
+    "Property 'Authorizers.MyLambdaAuthUpdated.Identity.Headers[0]' should be a string. ",
+    "Resource with id [MyApi2] is invalid. ",
+    "Property 'Authorizers.MyLambdaAuthUpdated.Identity.QueryStrings' should be a list."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 2. Resource with id [MyApi] is invalid. Property 'Authorizers.MyLambdaAuthUpdated.Identity.Headers[0]' should be a string. Resource with id [MyApi2] is invalid. Property 'Authorizers.MyLambdaAuthUpdated.Identity.QueryStrings' should be a list."
 }

tests/translator/output/error_api_authorizer_property_indentity_with_invalid_type.json

@@ -1,3 +1,15 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 4. ",
+    "Resource with id [MyApi] is invalid. ",
+    "Property 'Auth.Authorizers.MyLambdaAuthUpdated.Identity' should be a map. ",
+    "Resource with id [MyRestApi] is invalid. ",
+    "Property 'Authorizer.LambdaRequestIdentityNotObject.Identity' should be a map. ",
+    "Resource with id [MyRestApiInvalidHeadersItemType] is invalid. ",
+    "Property 'Auth.Authorizers.LambdaRequestIdentityNotObject.Identity.Headers[1]' should be a string. ",
+    "Resource with id [MyRestApiInvalidHeadersType] is invalid. ",
+    "Property 'Auth.Authorizers.LambdaRequestIdentityNotObject.Identity.Headers' should be a list."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 4. Resource with id [MyApi] is invalid. Property 'Auth.Authorizers.MyLambdaAuthUpdated.Identity' should be a map. Resource with id [MyRestApi] is invalid. Property 'Authorizer.LambdaRequestIdentityNotObject.Identity' should be a map. Resource with id [MyRestApiInvalidHeadersItemType] is invalid. Property 'Auth.Authorizers.LambdaRequestIdentityNotObject.Identity.Headers[1]' should be a string. Resource with id [MyRestApiInvalidHeadersType] is invalid. Property 'Auth.Authorizers.LambdaRequestIdentityNotObject.Identity.Headers' should be a list."
 }

tests/translator/output/error_api_duplicate_methods_same_path.json

@@ -1,4 +1,11 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 1. ",
+    "Resource with id [Function2] is invalid. ",
+    "Event with id [GetHtml] is invalid. ",
+    "API method \"post\" defined multiple times for path \"/add\"."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [Function2] is invalid. Event with id [GetHtml] is invalid. API method \"post\" defined multiple times for path \"/add\".",
   "errors": [
     {

tests/translator/output/error_api_event_import_vaule_reference.json

@@ -1,3 +1,10 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 1. ",
+    "Resource with id [Function] is invalid. ",
+    "Event with id [GetHtml] is invalid. ",
+    "Property 'RestApiId' should be a string."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [Function] is invalid. Event with id [GetHtml] is invalid. Property 'RestApiId' should be a string."
 }

tests/translator/output/error_api_event_ref_http_api.json

@@ -1,3 +1,10 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 1. ",
+    "Resource with id [MyFunction] is invalid. ",
+    "Event with id [Event1] is invalid. ",
+    "RestApiId must be a valid reference to an 'AWS::Serverless::Api' resource in same template."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [MyFunction] is invalid. Event with id [Event1] is invalid. RestApiId must be a valid reference to an 'AWS::Serverless::Api' resource in same template."
 }

tests/translator/output/error_api_event_ref_invalid_resource.json

@@ -1,3 +1,9 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 1. ",
+    "Resource with id [RestApi] is invalid. ",
+    "Attribute 'Properties' should be a map."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [RestApi] is invalid. Attribute 'Properties' should be a map."
 }

tests/translator/output/error_api_event_ref_nothing.json

@@ -1,3 +1,10 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 1. ",
+    "Resource with id [MyFunction] is invalid. ",
+    "Event with id [Event1] is invalid. ",
+    "RestApiId must be a valid reference to an 'AWS::Serverless::Api' resource in same template."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [MyFunction] is invalid. Event with id [Event1] is invalid. RestApiId must be a valid reference to an 'AWS::Serverless::Api' resource in same template."
 }

tests/translator/output/error_api_gateway_responses_nonnumeric_status_code.json

@@ -1,3 +1,9 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 1. ",
+    "Resource with id [ExplicitApi] is invalid. ",
+    "Property 'StatusCode' must be numeric"
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [ExplicitApi] is invalid. Property 'StatusCode' must be numeric"
 }

tests/translator/output/error_api_gateway_responses_responseparameter_invalid_type.json

@@ -1,3 +1,9 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 1. ",
+    "Resource with id [ExplicitApi] is invalid. ",
+    "Property 'GatewayResponses.UNAUTHORIZED.ResponseParameters' should be a map."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [ExplicitApi] is invalid. Property 'GatewayResponses.UNAUTHORIZED.ResponseParameters' should be a map."
 }

tests/translator/output/error_api_gateway_responses_unknown_responseparameter.json

@@ -1,3 +1,9 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 1. ",
+    "Resource with id [ExplicitApi] is invalid. ",
+    "Invalid gateway response parameter 'Footers'"
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [ExplicitApi] is invalid. Invalid gateway response parameter 'Footers'"
 }

tests/translator/output/error_api_gateway_responses_unknown_responseparameter_property.json

@@ -1,3 +1,9 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 1. ",
+    "Resource with id [ExplicitApi] is invalid. ",
+    "Invalid property 'SubStatusCode' in 'GatewayResponses' property 'UNAUTHORIZED'."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [ExplicitApi] is invalid. Invalid property 'SubStatusCode' in 'GatewayResponses' property 'UNAUTHORIZED'."
 }

tests/translator/output/error_api_invalid_auth.json

@@ -1,3 +1,51 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 20. ",
+    "Resource with id [AuthNotDictApi] is invalid. ",
+    "Property 'Auth' should be a map. ",
+    "Resource with id [AuthWithAdditionalPropertyApi] is invalid. ",
+    "Invalid value for 'Auth' property Resource with id [AuthWithDefinitionUriApi] is invalid. ",
+    "Auth works only with inline Swagger specified in 'DefinitionBody' property. ",
+    "Resource with id [AuthWithInvalidDefinitionBodyApi] is invalid. ",
+    "Unable to add Auth configuration because 'DefinitionBody' does not contain a valid Swagger definition. ",
+    "Resource with id [AuthWithMissingDefaultAuthorizerApi] is invalid. ",
+    "Unable to set DefaultAuthorizer because 'NotThere' was not defined in 'Authorizers'. ",
+    "Resource with id [AuthorizerNotDict] is invalid. ",
+    "Property 'Auth.Authorizers.MyCognitoAuthorizer' should be a map. ",
+    "Resource with id [AuthorizerWithBadDisableFunctionDefaultPermissionsType] is invalid. ",
+    "Property 'Authorizers.MyAuth.DisableFunctionDefaultPermissions' should be a boolean. ",
+    "Resource with id [AuthorizersNotDictApi] is invalid. ",
+    "Property 'Auth.Authorizers' should be a map. ",
+    "Resource with id [IntrinsicDefaultAuthorizerApi] is invalid. ",
+    "Property 'Auth.DefaultAuthorizer' should be a string. ",
+    "Resource with id [InvalidFunctionPayloadTypeApi] is invalid. ",
+    "MyLambdaAuthorizer Authorizer has invalid 'FunctionPayloadType': INVALID. ",
+    "Resource with id [MissingAuthorizerFn] is invalid. ",
+    "Event with id [GetRoot] is invalid. ",
+    "Unable to set Authorizer [UnspecifiedAuthorizer] on API method [get] for path [/] because it wasn't defined in the API's Authorizers. ",
+    "Resource with id [NoApiAuthorizerFn] is invalid. ",
+    "Event with id [GetRoot] is invalid. ",
+    "Unable to set Authorizer [MyAuth] on API method [get] for path [/] because the related API does not define any Authorizers. ",
+    "Resource with id [NoAuthFn] is invalid. ",
+    "Event with id [GetRoot] is invalid. ",
+    "Unable to set Authorizer [MyAuth] on API method [get] for path [/] because the related API does not define any Authorizers. ",
+    "Resource with id [NoAuthorizersFn] is invalid. ",
+    "Event with id [GetRoot] is invalid. ",
+    "Unable to set Authorizer [MyAuth] on API method [get] for path [/] because the related API does not define any Authorizers. ",
+    "Resource with id [NoDefaultAuthorizerWithNoneFn] is invalid. ",
+    "Event with id [GetRoot] is invalid. ",
+    "Unable to set Authorizer on API method [get] for path [/] because 'NONE' is only a valid value when a DefaultAuthorizer on the API is specified. ",
+    "Resource with id [NoIdentityOnRequestAuthorizer] is invalid. ",
+    "MyLambdaRequestAuthorizer Authorizer must specify Identity with at least one of Headers, QueryStrings, StageVariables, or Context. ",
+    "Resource with id [NoIdentitySourceOnRequestAuthorizer] is invalid. ",
+    "MyLambdaRequestAuthorizer Authorizer must specify Identity with at least one of Headers, QueryStrings, StageVariables, or Context. ",
+    "Resource with id [NonDictAuthorizerApi] is invalid. ",
+    "Property 'Auth.Authorizers.MyAuth' should be a map. ",
+    "Resource with id [NonDictAuthorizerRestApi] is invalid. ",
+    "Property 'Auth.Authorizers.MyAuth' should be a map. ",
+    "Resource with id [NonStringDefaultAuthorizerApi] is invalid. ",
+    "Property 'Auth.DefaultAuthorizer' should be a string."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 20. Resource with id [AuthNotDictApi] is invalid. Property 'Auth' should be a map. Resource with id [AuthWithAdditionalPropertyApi] is invalid. Invalid value for 'Auth' property Resource with id [AuthWithDefinitionUriApi] is invalid. Auth works only with inline Swagger specified in 'DefinitionBody' property. Resource with id [AuthWithInvalidDefinitionBodyApi] is invalid. Unable to add Auth configuration because 'DefinitionBody' does not contain a valid Swagger definition. Resource with id [AuthWithMissingDefaultAuthorizerApi] is invalid. Unable to set DefaultAuthorizer because 'NotThere' was not defined in 'Authorizers'. Resource with id [AuthorizerNotDict] is invalid. Property 'Auth.Authorizers.MyCognitoAuthorizer' should be a map. Resource with id [AuthorizerWithBadDisableFunctionDefaultPermissionsType] is invalid. Property 'Authorizers.MyAuth.DisableFunctionDefaultPermissions' should be a boolean. Resource with id [AuthorizersNotDictApi] is invalid. Property 'Auth.Authorizers' should be a map. Resource with id [IntrinsicDefaultAuthorizerApi] is invalid. Property 'Auth.DefaultAuthorizer' should be a string. Resource with id [InvalidFunctionPayloadTypeApi] is invalid. MyLambdaAuthorizer Authorizer has invalid 'FunctionPayloadType': INVALID. Resource with id [MissingAuthorizerFn] is invalid. Event with id [GetRoot] is invalid. Unable to set Authorizer [UnspecifiedAuthorizer] on API method [get] for path [/] because it wasn't defined in the API's Authorizers. Resource with id [NoApiAuthorizerFn] is invalid. Event with id [GetRoot] is invalid. Unable to set Authorizer [MyAuth] on API method [get] for path [/] because the related API does not define any Authorizers. Resource with id [NoAuthFn] is invalid. Event with id [GetRoot] is invalid. Unable to set Authorizer [MyAuth] on API method [get] for path [/] because the related API does not define any Authorizers. Resource with id [NoAuthorizersFn] is invalid. Event with id [GetRoot] is invalid. Unable to set Authorizer [MyAuth] on API method [get] for path [/] because the related API does not define any Authorizers. Resource with id [NoDefaultAuthorizerWithNoneFn] is invalid. Event with id [GetRoot] is invalid. Unable to set Authorizer on API method [get] for path [/] because 'NONE' is only a valid value when a DefaultAuthorizer on the API is specified. Resource with id [NoIdentityOnRequestAuthorizer] is invalid. MyLambdaRequestAuthorizer Authorizer must specify Identity with at least one of Headers, QueryStrings, StageVariables, or Context. Resource with id [NoIdentitySourceOnRequestAuthorizer] is invalid. MyLambdaRequestAuthorizer Authorizer must specify Identity with at least one of Headers, QueryStrings, StageVariables, or Context. Resource with id [NonDictAuthorizerApi] is invalid. Property 'Auth.Authorizers.MyAuth' should be a map. Resource with id [NonDictAuthorizerRestApi] is invalid. Property 'Auth.Authorizers.MyAuth' should be a map. Resource with id [NonStringDefaultAuthorizerApi] is invalid. Property 'Auth.DefaultAuthorizer' should be a string."
 }

tests/translator/output/error_api_invalid_auth_identity_cognito.json

@@ -1,4 +1,10 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 1. ",
+    "Resource with id [ServerlessApi] is invalid. ",
+    "Auth.Authorizers.<Authorizer>.Identity must be a dict (LambdaTokenAuthorizationIdentity, LambdaRequestAuthorizationIdentity or CognitoAuthorizationIdentity)."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [ServerlessApi] is invalid. Auth.Authorizers.<Authorizer>.Identity must be a dict (LambdaTokenAuthorizationIdentity, LambdaRequestAuthorizationIdentity or CognitoAuthorizationIdentity).",
   "errors": [
     {

tests/translator/output/error_api_invalid_auth_identity_lambda_request.json

@@ -1,4 +1,10 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 1. ",
+    "Resource with id [ServerlessApi] is invalid. ",
+    "Auth.Authorizers.<Authorizer>.Identity must be a dict (LambdaTokenAuthorizationIdentity, LambdaRequestAuthorizationIdentity or CognitoAuthorizationIdentity)."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [ServerlessApi] is invalid. Auth.Authorizers.<Authorizer>.Identity must be a dict (LambdaTokenAuthorizationIdentity, LambdaRequestAuthorizationIdentity or CognitoAuthorizationIdentity).",
   "errors": [
     {

tests/translator/output/error_api_invalid_definitionbody.json

@@ -1,3 +1,9 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 1. ",
+    "Resource with id [ApiWithInvalidBodyType] is invalid. ",
+    "Property 'DefinitionBody' should be a map."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [ApiWithInvalidBodyType] is invalid. Property 'DefinitionBody' should be a map."
 }

tests/translator/output/error_api_invalid_definitionuri.json

@@ -1,4 +1,12 @@
 {
+  "_autoGeneratedBreakdownErrorMessage": [
+    "Invalid Serverless Application Specification document. ",
+    "Number of errors found: 2. ",
+    "Resource with id [Api] is invalid. ",
+    "'DefinitionUri' is not a valid S3 Uri of the form 's3://bucket/key' with optional versionId query parameter. ",
+    "Resource with id [ApiWithBodyAndDefinitionUri] is invalid. ",
+    "Specify either 'DefinitionUri' or 'DefinitionBody' property and not both."
+  ],
   "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 2. Resource with id [Api] is invalid. 'DefinitionUri' is not a valid S3 Uri of the form 's3://bucket/key' with optional versionId query parameter. Resource with id [ApiWithBodyAndDefinitionUri] is invalid. Specify either 'DefinitionUri' or 'DefinitionBody' property and not both.",
   "errors": [
     {