From 9cb84ea5631a361e31d28e7ae5f780d25bedaa46 Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Mon, 1 Nov 2021 12:47:40 -0400 Subject: [PATCH 1/2] feat: add elasticache replication group and unit test framework --- justfile | 7 + tests/test_tfdevops.py | 5 - tests/unit/conftest.py | 50 +++ tests/unit/data/elasticache.json | 85 +++++ .../schema.elasticache_replication_group.json | 315 ++++++++++++++++++ tests/unit/test_resources.py | 10 + tfdevops/cli.py | 63 +++- 7 files changed, 513 insertions(+), 22 deletions(-) create mode 100644 justfile delete mode 100644 tests/test_tfdevops.py create mode 100644 tests/unit/conftest.py create mode 100644 tests/unit/data/elasticache.json create mode 100644 tests/unit/data/schema.elasticache_replication_group.json create mode 100644 tests/unit/test_resources.py diff --git a/justfile b/justfile new file mode 100644 index 0000000..4f1a674 --- /dev/null +++ b/justfile @@ -0,0 +1,7 @@ + + +lint: + pre-commit run --all-files + +test: + pytest -v tests/unit diff --git a/tests/test_tfdevops.py b/tests/test_tfdevops.py deleted file mode 100644 index 03044eb..0000000 --- a/tests/test_tfdevops.py +++ /dev/null @@ -1,5 +0,0 @@ -from tfdevops.cli import TF_CFN_MAP, Translator - - -def test_translator_map(): - assert set(Translator.get_translator_map()) == set(TF_CFN_MAP) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 0000000..fd1797a --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,50 @@ +import json +from pathlib import Path + +import boto3 +import jsonschema +import pytest + + +@pytest.fixture() +def validator(): + def schema_validate(translator, resource): + schema_path = f"schema.{translator.tf_type}.json" + schema = load_data(schema_path) + if schema is None: + cfn = boto3.client("cloudformation") + rtype = cfn.describe_type(TypeName=translator.cfn_type, Type="RESOURCE") + schema = json.loads(rtype["Schema"]) + (Path(__file__).parent / "data" / schema_path).write_text( + json.dumps(schema, indent=2) + ) + + props = set(resource) + sprops = set(schema["properties"].keys()) + unknown = props.difference(sprops) + if unknown: + raise KeyError("unknown resource keys %s" % (", ".join(unknown))) + + validator = jsonschema.Draft7Validator(schema) + + errors = list(validator.iter_errors(resource)) + if errors: + print("%s errors %d" % (translator.cfn_type, len(errors))) + + for e in errors: + print("Resource %s error:\n %s" % (translator.cfn_type, str(e))) + + if errors: + raise ValueError( + f"resource type {translator.cfn_type} had translation errors" + ) + + return schema_validate + + +def load_data(filename): + path = Path(__file__).parent / "data" / filename + if not path.exists(): + return None + with open(path) as f: + return json.load(f) diff --git a/tests/unit/data/elasticache.json b/tests/unit/data/elasticache.json new file mode 100644 index 0000000..2fbac5b --- /dev/null +++ b/tests/unit/data/elasticache.json @@ -0,0 +1,85 @@ + { + "address": "aws_elasticache_replication_group.buffer", + "mode": "managed", + "type": "aws_elasticache_replication_group", + "name": "buffer", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 1, + "values": { + "apply_immediately": true, + "arn": "arn:aws:elasticache:us-east-2:112233445566:replicationgroup:stack-sample-buffer", + "at_rest_encryption_enabled": true, + "auth_token": "", + "auto_minor_version_upgrade": true, + "automatic_failover_enabled": false, + "availability_zones": null, + "cluster_enabled": false, + "cluster_mode": [ + { + "num_node_groups": 1, + "replicas_per_node_group": 0 + } + ], + "configuration_endpoint_address": null, + "engine": "redis", + "engine_version": "6.x", + "engine_version_actual": "6.0.5", + "final_snapshot_identifier": null, + "global_replication_group_id": null, + "id": "stack-sample-buffer", + "kms_key_id": "arn:aws:kms:us-east-2:112233445566:key/a33e6586-615d-4214-b2cc-17c3d48d7aea", + "maintenance_window": "mon:06:00-mon:07:00", + "member_clusters": [ + "stack-sample-buffer-001" + ], + "multi_az_enabled": false, + "node_type": "cache.m6g.large", + "notification_topic_arn": null, + "number_cache_clusters": 1, + "parameter_group_name": "default.redis6.x", + "port": 6379, + "primary_endpoint_address": "master.stack-sample-buffer.iyyvzj.use2.cache.amazonaws.com", + "reader_endpoint_address": "replica.stack-sample-buffer.iyyvzj.use2.cache.amazonaws.com", + "replication_group_description": "Elasticache cluster with encrypted redis", + "replication_group_id": "stack-sample-buffer", + "security_group_ids": [ + "sg-0168ebe76be6927ce" + ], + "security_group_names": [], + "snapshot_arns": null, + "snapshot_name": null, + "snapshot_retention_limit": 0, + "snapshot_window": "02:30-03:30", + "subnet_group_name": "stack-sample-buffer", + "tags": {}, + "tags_all": { + "App": "Sample" + }, + "timeouts": null, + "transit_encryption_enabled": true + }, + "sensitive_values": { + "cluster_mode": [ + {} + ], + "member_clusters": [ + false + ], + "security_group_ids": [ + false + ], + "security_group_names": [], + "tags": {}, + "tags_all": {} + }, + "depends_on": [ + "data.aws_region.current", + "aws_elasticache_subnet_group.buffer", + "aws_iam_role.app_role", + "aws_kms_key.cache_kms_encrypt", + "aws_security_group.db", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.app_role_assume_role_policy", + "data.aws_iam_policy_document.cache_kms_policy" + ] + } diff --git a/tests/unit/data/schema.elasticache_replication_group.json b/tests/unit/data/schema.elasticache_replication_group.json new file mode 100644 index 0000000..65f95df --- /dev/null +++ b/tests/unit/data/schema.elasticache_replication_group.json @@ -0,0 +1,315 @@ +{ + "typeName": "AWS::ElastiCache::ReplicationGroup", + "description": "Resource Type definition for AWS::ElastiCache::ReplicationGroup", + "additionalProperties": false, + "properties": { + "PreferredCacheClusterAZs": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "PrimaryEndPointPort": { + "type": "string" + }, + "CacheSecurityGroupNames": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "ReaderEndPointPort": { + "type": "string" + }, + "NodeGroupConfiguration": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/NodeGroupConfiguration" + } + }, + "SnapshotArns": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "ConfigurationEndPointPort": { + "type": "string" + }, + "Port": { + "type": "integer" + }, + "ReadEndPointPortsList": { + "type": "array", + "uniqueItems": false, + "items": { + "type": "string" + } + }, + "NumNodeGroups": { + "type": "integer" + }, + "NotificationTopicArn": { + "type": "string" + }, + "SnapshotName": { + "type": "string" + }, + "AutomaticFailoverEnabled": { + "type": "boolean" + }, + "ReplicasPerNodeGroup": { + "type": "integer" + }, + "ReplicationGroupDescription": { + "type": "string" + }, + "ReaderEndPointAddress": { + "type": "string" + }, + "MultiAZEnabled": { + "type": "boolean" + }, + "TransitEncryptionEnabled": { + "type": "boolean" + }, + "ReplicationGroupId": { + "type": "string" + }, + "Engine": { + "type": "string" + }, + "Tags": { + "type": "array", + "uniqueItems": false, + "items": { + "$ref": "#/definitions/Tag" + } + }, + "NumCacheClusters": { + "type": "integer" + }, + "PrimaryEndPointAddress": { + "type": "string" + }, + "GlobalReplicationGroupId": { + "type": "string" + }, + "ConfigurationEndPointAddress": { + "type": "string" + }, + "EngineVersion": { + "type": "string" + }, + "KmsKeyId": { + "type": "string" + }, + "CacheSubnetGroupName": { + "type": "string" + }, + "CacheParameterGroupName": { + "type": "string" + }, + "PreferredMaintenanceWindow": { + "type": "string" + }, + "PrimaryClusterId": { + "type": "string" + }, + "ReadEndPointPorts": { + "type": "string" + }, + "AtRestEncryptionEnabled": { + "type": "boolean" + }, + "AutoMinorVersionUpgrade": { + "type": "boolean" + }, + "SecurityGroupIds": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "SnapshotWindow": { + "type": "string" + }, + "CacheNodeType": { + "type": "string" + }, + "SnapshotRetentionLimit": { + "type": "integer" + }, + "ReadEndPointAddressesList": { + "type": "array", + "uniqueItems": false, + "items": { + "type": "string" + } + }, + "SnapshottingClusterId": { + "type": "string" + }, + "UserGroupIds": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "AuthToken": { + "type": "string" + }, + "LogDeliveryConfigurations": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/LogDeliveryConfigurationRequest" + } + }, + "ReadEndPointAddresses": { + "type": "string" + } + }, + "definitions": { + "LogDeliveryConfigurationRequest": { + "type": "object", + "additionalProperties": false, + "properties": { + "LogType": { + "type": "string" + }, + "LogFormat": { + "type": "string" + }, + "DestinationType": { + "type": "string" + }, + "DestinationDetails": { + "$ref": "#/definitions/DestinationDetails" + } + }, + "required": [ + "LogFormat", + "LogType", + "DestinationType", + "DestinationDetails" + ] + }, + "KinesisFirehoseDestinationDetails": { + "type": "object", + "additionalProperties": false, + "properties": { + "DeliveryStream": { + "type": "string" + } + }, + "required": [ + "DeliveryStream" + ] + }, + "CloudWatchLogsDestinationDetails": { + "type": "object", + "additionalProperties": false, + "properties": { + "LogGroup": { + "type": "string" + } + }, + "required": [ + "LogGroup" + ] + }, + "NodeGroupConfiguration": { + "type": "object", + "additionalProperties": false, + "properties": { + "Slots": { + "type": "string" + }, + "PrimaryAvailabilityZone": { + "type": "string" + }, + "ReplicaAvailabilityZones": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "NodeGroupId": { + "type": "string" + }, + "ReplicaCount": { + "type": "integer" + } + } + }, + "Tag": { + "type": "object", + "additionalProperties": false, + "properties": { + "Value": { + "type": "string" + }, + "Key": { + "type": "string" + } + }, + "required": [ + "Value", + "Key" + ] + }, + "DestinationDetails": { + "type": "object", + "additionalProperties": false, + "properties": { + "CloudWatchLogsDetails": { + "$ref": "#/definitions/CloudWatchLogsDestinationDetails" + }, + "KinesisFirehoseDetails": { + "$ref": "#/definitions/KinesisFirehoseDestinationDetails" + } + } + } + }, + "required": [ + "ReplicationGroupDescription" + ], + "createOnlyProperties": [ + "/properties/KmsKeyId", + "/properties/Port", + "/properties/SnapshotArns", + "/properties/SnapshotName", + "/properties/TransitEncryptionEnabled", + "/properties/CacheSubnetGroupName", + "/properties/AtRestEncryptionEnabled", + "/properties/ReplicationGroupId", + "/properties/GlobalReplicationGroupId", + "/properties/ReplicasPerNodeGroup", + "/properties/Engine", + "/properties/PreferredCacheClusterAZs" + ], + "primaryIdentifier": [ + "/properties/ReplicationGroupId" + ], + "readOnlyProperties": [ + "/properties/ConfigurationEndPoint.Address", + "/properties/PrimaryEndPoint.Address", + "/properties/PrimaryEndPoint.Port", + "/properties/ReaderEndPoint.Address", + "/properties/ConfigurationEndPoint.Port", + "/properties/ReadEndPoint.Addresses.List", + "/properties/ReadEndPoint.Ports.List", + "/properties/ReaderEndPoint.Port", + "/properties/ReadEndPoint.Addresses", + "/properties/ReadEndPoint.Ports", + "/properties/ReplicationGroupId" + ] +} \ No newline at end of file diff --git a/tests/unit/test_resources.py b/tests/unit/test_resources.py new file mode 100644 index 0000000..1500f45 --- /dev/null +++ b/tests/unit/test_resources.py @@ -0,0 +1,10 @@ +from conftest import load_data +from tfdevops.cli import Translator + + +def test_elasticache_replication_group(validator): + + translator = Translator.get_translator("elasticache_replication_group")() + resource = load_data("elasticache.json") + props = translator.get_properties(resource) + validator(translator, props) diff --git a/tfdevops/cli.py b/tfdevops/cli.py index 2b17d96..7bb6065 100644 --- a/tfdevops/cli.py +++ b/tfdevops/cli.py @@ -388,7 +388,6 @@ def cfn(module, template, resources, types, s3_path, state_file): overflow cloudformation's api limits on templates (50k). """ state = get_state_resources(module, state_file) - type_map = TF_CFN_MAP ctemplate = { "AWSTemplateFormatVersion": "2010-09-09", @@ -403,11 +402,12 @@ def cfn(module, template, resources, types, s3_path, state_file): provider, k = k.split("_", 1) if types and k not in types: continue - if k not in type_map: + if k not in translators: log.debug("no cfn type for tf %s" % k) continue - cfn_type = type_map[k] + translator_class = translators.get(k) + cfn_type = translator_class.cfn_type if not translator_class: log.debug("no translator for %s" % k) continue @@ -534,19 +534,6 @@ def format_template_url(client, s3_path): return url.format(bucket=bucket, key=key, version_id=version_id, region=region) -TF_CFN_MAP = { - "cloudwatch_event_rule": "AWS::Events::Rule", - "db_instance": "AWS::RDS::DBInstance", - "sns_topic": "AWS::SNS::Topic", - "sqs_queue": "AWS::SQS::Queue", - "lambda_function": "AWS::Lambda::Function", - "sfn_state_machine": "AWS::StepFunctions::StateMachine", - "cloudwatch_event_rule": "AWS::Events::Rule", - "ecs_service": "AWS::ECS::Service", - "dynamodb_table": "AWS::DynamoDB::Table", -} - - class Translator: id = None @@ -555,9 +542,13 @@ class Translator: rename = {} flatten = () - def __init__(self, config): + def __init__(self, config=None): self.config = config + @classmethod + def get_translator(cls, tf_type): + return cls.get_translator_map()[tf_type] + @classmethod def get_translator_map(cls): d = {} @@ -620,6 +611,8 @@ def camel(self, d): class EventRuleTranslator(Translator): tf_type = "cloudwatch_event_rule" + cfn_type = "AWS::Events::Rule" + id = "Name" def get_properties(self, r): @@ -635,6 +628,7 @@ def get_properties(self, r): class DbInstance(Translator): tf_type = "db_instance" + cfn_type = "AWS::RDS::DBInstance" id = "DBInstanceIdentifier" strip = ( "hosted_zone_id", @@ -677,9 +671,34 @@ def get_properties(self, tf): return cfr +class ElasticacheReplicationGroup(Translator): + + tf_type = "elasticache_replication_group" + cfn_type = "AWS::ElastiCache::ReplicationGroup" + + id = "ReplicationGroupId" + rename = { + "subnet_group_name": "CacheSubnetGroupName", + "maintenance_window": "PreferredMaintenanceWindow", + "number_cache_clusters": "NumCacheClusters", + "node_type": "CacheNodeType", + "parameter_group_name": "CacheParameterGroupName", + } + strip = ( + "primary_endpoint_address", + "reader_endpoint_address", + "member_clusters", + "engine_version_actual", + "apply_immediately", + "cluster_mode", + ) + + class EcsService(Translator): tf_type = "ecs_service" + cfn_type = "AWS::ECS::Service" + id = "ServiceName" flatten = ("network_configuration", "deployment_controller") rename = {"iam_role": "Role", "enable_ecs_managed_tags": "EnableECSManagedTags"} @@ -707,6 +726,8 @@ def get_properties(self, tf): class Sqs(Translator): tf_type = "sqs_queue" + cfn_type = "AWS::SQS::Queue" + id = "QueueUrl" strip = ("url", "policy", "fifo_throughput_limit", "deduplication_scope") rename = { @@ -730,6 +751,8 @@ def get_properties(self, tf): class Topic(Translator): tf_type = "sns_topic" + cfn_type = "AWS::SNS::Topic" + id = "TopicArn" strip = ("policy", "owner") rename = {"name": "TopicName"} @@ -741,6 +764,8 @@ def get_identity(self, r): class Lambda(Translator): tf_type = "lambda_function" + cfn_type = "AWS::Lambda::Function" + id = "FunctionName" flatten = ("environment", "tracing_config", "vpc_config") strip = ( @@ -778,6 +803,8 @@ def get_properties(self, tfr): class StateMachine(Translator): tf_type = "sfn_state_machine" + cfn_type = "AWS::StepFunctions::StateMachine" + id = "Arn" strip = ( "definition", @@ -818,6 +845,8 @@ def get_properties(self, tf): class DynamodbTable(Translator): tf_type = "dynamodb_table" + cfn_type = "AWS::DynamoDB::Table" + id = "TableName" rename = {"name": "TableName"} strip = ( From 2d3d239dc4e63ac019d9eb2963bb594336139eec Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Mon, 1 Nov 2021 13:17:51 -0400 Subject: [PATCH 2/2] rename validator to validate --- tests/unit/conftest.py | 2 +- tests/unit/test_resources.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index fd1797a..b99838f 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -7,7 +7,7 @@ @pytest.fixture() -def validator(): +def validate(): def schema_validate(translator, resource): schema_path = f"schema.{translator.tf_type}.json" schema = load_data(schema_path) diff --git a/tests/unit/test_resources.py b/tests/unit/test_resources.py index 1500f45..28a4ab6 100644 --- a/tests/unit/test_resources.py +++ b/tests/unit/test_resources.py @@ -2,9 +2,8 @@ from tfdevops.cli import Translator -def test_elasticache_replication_group(validator): - +def test_elasticache_replication_group(validate): translator = Translator.get_translator("elasticache_replication_group")() resource = load_data("elasticache.json") props = translator.get_properties(resource) - validator(translator, props) + validate(translator, props)