diff --git a/Makefile b/Makefile index a074f65ba9..2b2ef25bb4 100755 --- a/Makefile +++ b/Makefile @@ -38,8 +38,6 @@ lint: mypy --strict samtranslator bin # Linter performs static analysis to catch latent bugs pylint --rcfile .pylintrc samtranslator - # Ensure templates adhere to JSON schema - bin/validate_schema.py prepare-companion-stack: pytest -v --no-cov integration/setup -m setup diff --git a/bin/validate_schema.py b/bin/validate_schema.py deleted file mode 100755 index 1a31384f03..0000000000 --- a/bin/validate_schema.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python - -import json -from pathlib import Path -from typing import Iterator - -from cfn_flip import to_json # type: ignore -from jsonschema import validate - -SCHEMA = json.loads(Path("samtranslator/schema/schema.json").read_bytes()) - - -def get_templates() -> Iterator[Path]: - paths = ( - list(Path("tests/translator/input").glob("**/*.yaml")) - + list(Path("tests/translator/input").glob("**/*.yml")) - + list(Path("tests/validator/input").glob("**/*.yaml")) - + list(Path("tests/validator/input").glob("**/*.yml")) - + list(Path("integration/resources/templates").glob("**/*.yaml")) - + list(Path("integration/resources/templates").glob("**/*.yml")) - ) - # TODO: Enable (most likely) everything but error_ - skips = [ - "error_", - "unsupported_resources", - "resource_with_invalid_type", - "state_machine_with_null_events", - "state_machine_with_cwe", # Doesn't match schema at all... - "function_with_null_events", - "function_with_event_bridge_rule_state", # Doesn't match schema at all... - "sns_existing_sqs", # 8 is not of type string - "eventbridgerule_with_dlq", # Doesn't match schema at all... - "sns_outside_sqs", # 8 is not of type string - "function_with_cwe_dlq_and_retry_policy", # Doesn't match schema at all... - "function_with_cwe_dlq_generated", # Doesn't match schema at all... - "function_with_request_parameters", # RequestParameters don't match documentation. Documentation and its example don't match either - "api_with_request_parameters_openapi", # RequestParameters don't match documentation. Documentation and its example don't match either - "api_with_aws_iam_auth_overrides", # null for invokeRole - "eventbridgerule", # missing required field 'Patterns' - "self_managed_kafka_with_intrinsics", # 'EnableValue' is of type bool but defined as string - "api_with_resource_policy_global", # 'ResourcePolicy CustomStatements' output expects a List - "api_with_resource_policy", # 'ResourcePolicy CustomStatements' output expects a List - "api_with_if_conditional_with_resource_policy", # 'ResourcePolicy CustomStatements' output expects a List - "api_rest_paths_with_if_condition_swagger", # 'EnableSimpleResponses' and 'AuthorizerPayloadFormatVersion' not defined in documentation - "api_rest_paths_with_if_condition_openapi", # 'EnableSimpleResponses' and 'AuthorizerPayloadFormatVersion' not defined in documentation - "state_machine_with_api_authorizer_maximum", # 'UserPoolArn' expects to be a string, but received list - "api_with_auth_all_maximum", # 'UserPoolArn' expects to be a string, but received list - "api_with_auth_and_conditions_all_max", # 'UserPoolArn' expects to be a string, but received list - "api_with_auth_all_maximum_openapi_3", # 'UserPoolArn' expects to be a string, but received list - "api_with_authorizers_max_openapi", # 'UserPoolArn' expects to be a string, but received list - "api_with_authorizers_max", # 'UserPoolArn' expects to be a string, but received list - "api_with_any_method_in_swagger", # Missing required field 'FunctionArn' - "api_with_cors_and_only_headers", # 'AllowOrigins' is required field - "api_with_cors_and_only_methods", # 'AllowOrigins' is required field - "implicit_api_with_auth_and_conditions_max", # 'UserPoolArn' expects to be a string, but received list - ] - - def should_skip(s: str) -> bool: - for skip in skips: - if skip in s: - return True - return False - - for path in paths: - if not should_skip(str(path)): - yield path - - -def main() -> None: - for path in get_templates(): - print(f"Checking {path}") - obj = json.loads(to_json(path.read_bytes())) - validate(obj, schema=SCHEMA) - - -if __name__ == "__main__": - main() diff --git a/tests/schema/__init__.py b/tests/schema/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/schema/test_validate_schema.py b/tests/schema/test_validate_schema.py new file mode 100644 index 0000000000..f405dda477 --- /dev/null +++ b/tests/schema/test_validate_schema.py @@ -0,0 +1,90 @@ +import json +import pytest +import os +import itertools + +from pathlib import Path +from typing import Iterator +from unittest import TestCase +from cfn_flip import to_json # type: ignore +from jsonschema import validate +from jsonschema.exceptions import ValidationError +from parameterized import parameterized + + +SCHEMA = json.loads(Path("samtranslator/schema/schema.json").read_bytes()) + +# TODO: Enable (most likely) everything but 'error_*' and 'basic_schema_validation_failure' +SKIPPED_TESTS = [ + "error_", + "unsupported_resources", + "resource_with_invalid_type", + "state_machine_with_null_events", + "state_machine_with_cwe", # Doesn't match schema at all... + "function_with_null_events", + "function_with_event_bridge_rule_state", # Doesn't match schema at all... + "sns_existing_sqs", # 8 is not of type string + "eventbridgerule_with_dlq", # Doesn't match schema at all... + "sns_outside_sqs", # 8 is not of type string + "function_with_cwe_dlq_and_retry_policy", # Doesn't match schema at all... + "function_with_cwe_dlq_generated", # Doesn't match schema at all... + "function_with_request_parameters", # RequestParameters don't match documentation. Documentation and its example don't match either + "api_with_request_parameters_openapi", # RequestParameters don't match documentation. Documentation and its example don't match either + "api_with_aws_iam_auth_overrides", # null for invokeRole + "eventbridgerule", # missing required field 'Patterns' + "self_managed_kafka_with_intrinsics", # 'EnableValue' is of type bool but defined as string + "api_with_resource_policy_global", # 'ResourcePolicy CustomStatements' output expects a List + "api_with_resource_policy", # 'ResourcePolicy CustomStatements' output expects a List + "api_with_if_conditional_with_resource_policy", # 'ResourcePolicy CustomStatements' output expects a List + "api_rest_paths_with_if_condition_swagger", # 'EnableSimpleResponses' and 'AuthorizerPayloadFormatVersion' not defined in documentation + "api_rest_paths_with_if_condition_openapi", # 'EnableSimpleResponses' and 'AuthorizerPayloadFormatVersion' not defined in documentation + "state_machine_with_api_authorizer_maximum", # 'UserPoolArn' expects to be a string, but received list + "api_with_auth_all_maximum", # 'UserPoolArn' expects to be a string, but received list + "api_with_auth_and_conditions_all_max", # 'UserPoolArn' expects to be a string, but received list + "api_with_auth_all_maximum_openapi_3", # 'UserPoolArn' expects to be a string, but received list + "api_with_authorizers_max_openapi", # 'UserPoolArn' expects to be a string, but received list + "api_with_authorizers_max", # 'UserPoolArn' expects to be a string, but received list + "api_with_any_method_in_swagger", # Missing required field 'FunctionArn' + "api_with_cors_and_only_headers", # 'AllowOrigins' is required field + "api_with_cors_and_only_methods", # 'AllowOrigins' is required field + "implicit_api_with_auth_and_conditions_max", # 'UserPoolArn' expects to be a string, but received list +] + + +def should_skip_test(s: str) -> bool: + for test in SKIPPED_TESTS: + if test in s: + return True + return False + + +def get_all_test_templates(): + return ( + list(Path("tests/translator/input").glob("**/*.yaml")) + + list(Path("tests/translator/input").glob("**/*.yml")) + + list(Path("tests/validator/input").glob("**/*.yaml")) + + list(Path("tests/validator/input").glob("**/*.yml")) + + list(Path("integration/resources/templates").glob("**/*.yaml")) + + list(Path("integration/resources/templates").glob("**/*.yml")) + ) + + +SCHEMA_VALIDATION_TESTS = [str(f) for f in get_all_test_templates() if not should_skip_test(str(f))] + + +class TestValidateSchema(TestCase): + @parameterized.expand(itertools.product(SCHEMA_VALIDATION_TESTS)) + def test_validate_schema(self, testcase): + obj = json.loads(to_json(Path(testcase).read_bytes())) + validate(obj, schema=SCHEMA) + + @parameterized.expand( + [ + "tests/translator/input/error_schema_validation_wrong_property.yaml", + "tests/translator/input/error_schema_validation_wrong_type.yaml", + ] + ) + def test_validate_schema_error(self, testcase): + obj = json.loads(to_json(Path(testcase).read_bytes())) + with pytest.raises(ValidationError): + validate(obj, schema=SCHEMA) diff --git a/tests/translator/input/error_schema_validation_wrong_property.yaml b/tests/translator/input/error_schema_validation_wrong_property.yaml new file mode 100644 index 0000000000..8733bdbd3a --- /dev/null +++ b/tests/translator/input/error_schema_validation_wrong_property.yaml @@ -0,0 +1,25 @@ +Resources: + MyFunction: + Type: AWS::Serverless::Function + Properties: + Runtime: nodejs14.x + Handler: index.handler + InlineCode: | + const AWS = require('aws-sdk'); + exports.handler = async (event) => { + console.log(JSON.stringify(event)); + }; + + MyBucket: + Type: AWS::S3::Bucket + + MyConnector: + Type: AWS::Serverless::Connector + Properties: + Source: + Id: MyFunction + Desitation: # wrong spelling + Id: MyBucket + Permissions: + - Read + - Write diff --git a/tests/translator/input/error_schema_validation_wrong_type.yaml b/tests/translator/input/error_schema_validation_wrong_type.yaml new file mode 100644 index 0000000000..d30bf89263 --- /dev/null +++ b/tests/translator/input/error_schema_validation_wrong_type.yaml @@ -0,0 +1,24 @@ +Resources: + MyFunction: + Type: AWS::Serverless::Function + Properties: + Runtime: nodejs14.x + Handler: index.handler + InlineCode: | + const AWS = require('aws-sdk'); + exports.handler = async (event) => { + console.log(JSON.stringify(event)); + }; + MyBucket: + Type: AWS::S3::Bucket + + MyConnector: + Type: AWS::Serverless::Connector + Properties: + Source: + Id: MyFunction + Destination: + Id: 42 # wrong type + Permissions: + - Read + - Write diff --git a/tests/translator/output/error_schema_validation_wrong_property.json b/tests/translator/output/error_schema_validation_wrong_property.json new file mode 100644 index 0000000000..dfd06f4ace --- /dev/null +++ b/tests/translator/output/error_schema_validation_wrong_property.json @@ -0,0 +1,3 @@ +{ + "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [MyConnector] is invalid. property Desitation not defined for resource of type AWS::Serverless::Connector" +} diff --git a/tests/translator/output/error_schema_validation_wrong_type.json b/tests/translator/output/error_schema_validation_wrong_type.json new file mode 100644 index 0000000000..d2ef275b33 --- /dev/null +++ b/tests/translator/output/error_schema_validation_wrong_type.json @@ -0,0 +1,3 @@ +{ + "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [MyConnector] is invalid. 'Id' is missing or not a string." +}