From 8e57ee31bfaa09a681a4615eb0032448a4d2feef Mon Sep 17 00:00:00 2001 From: laxmiprasanna-gunna Date: Thu, 13 Dec 2018 10:22:11 +0530 Subject: [PATCH 1/3] Add Dynamo DB support --- README.md | 1 + lib/terraforming.rb | 2 + lib/terraforming/cli.rb | 5 + lib/terraforming/resource/dynamo_db.rb | 285 ++++++++++++++++++ lib/terraforming/template/tf/dynamo_db.erb | 64 ++++ spec/lib/terraforming/cli_spec.rb | 7 + .../terraforming/resource/dynamo_db_spec.rb | 145 +++++++++ terraforming.gemspec | 1 + 8 files changed, 510 insertions(+) create mode 100644 lib/terraforming/resource/dynamo_db.rb create mode 100644 lib/terraforming/template/tf/dynamo_db.erb create mode 100644 spec/lib/terraforming/resource/dynamo_db_spec.rb diff --git a/README.md b/README.md index 9251ba9e..98769fd8 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,7 @@ Commands: terraforming dbpg # Database Parameter Group terraforming dbsg # Database Security Group terraforming dbsn # Database Subnet Group + terraforming ddb # Dynamo DB terraforming ec2 # EC2 terraforming ecc # ElastiCache Cluster terraforming ecsn # ElastiCache Subnet Group diff --git a/lib/terraforming.rb b/lib/terraforming.rb index 27f7eee2..c4de94ba 100644 --- a/lib/terraforming.rb +++ b/lib/terraforming.rb @@ -1,5 +1,6 @@ require "aws-sdk-autoscaling" require "aws-sdk-cloudwatch" +require "aws-sdk-dynamodb" require "aws-sdk-ec2" require "aws-sdk-efs" require "aws-sdk-elasticache" @@ -29,6 +30,7 @@ require "terraforming/resource/db_parameter_group" require "terraforming/resource/db_security_group" require "terraforming/resource/db_subnet_group" +require "terraforming/resource/dynamo_db" require "terraforming/resource/ec2" require "terraforming/resource/eip" require "terraforming/resource/elasti_cache_cluster" diff --git a/lib/terraforming/cli.rb b/lib/terraforming/cli.rb index bacc38b0..5c290d5b 100644 --- a/lib/terraforming/cli.rb +++ b/lib/terraforming/cli.rb @@ -40,6 +40,11 @@ def dbsn execute(Terraforming::Resource::DBSubnetGroup, options) end + desc "ddb", "Dynamo DB" + def ddb + execute(Terraforming::Resource::DynamoDb, options) + end + desc "ec2", "EC2" def ec2 execute(Terraforming::Resource::EC2, options) diff --git a/lib/terraforming/resource/dynamo_db.rb b/lib/terraforming/resource/dynamo_db.rb new file mode 100644 index 00000000..a8a49c1f --- /dev/null +++ b/lib/terraforming/resource/dynamo_db.rb @@ -0,0 +1,285 @@ +module Terraforming + module Resource + class DynamoDb + include Terraforming::Util + def self.tf(client: Aws::DynamoDB::Client.new) + self.new(client).tf + end + + def self.tfstate(client: Aws::DynamoDB::Client.new) + self.new(client).tfstate + end + + def initialize(client) + @client = client + end + + def tf + apply_template(@client, "tf/dynamo_db") + end + + def tfstate + tables.inject({}) do |resources, dynamo_db_table| + attributes = { + "arn" => dynamo_db_table["table_arn"], + "id" => dynamo_db_table["table_name"], + "name" => dynamo_db_table["table_name"], + "read_capacity" => dynamo_db_table["provisioned_throughput"]["read_capacity_units"].to_s, + "stream_arn" => dynamo_db_table["latest_stream_arn"].to_s, + "stream_label" => dynamo_db_table["latest_stream_label"].to_s, + "write_capacity" => dynamo_db_table["provisioned_throughput"]["write_capacity_units"].to_s + } + + attributes.merge!(attribute_definitions(dynamo_db_table)) + attributes.merge!(global_indexes(dynamo_db_table)) + attributes.merge!(local_indexes(dynamo_db_table)) + attributes.merge!(key_schema(dynamo_db_table)) + attributes.merge!(point_in_time_summary(dynamo_db_table)) + attributes.merge!(sse_description(dynamo_db_table)) + attributes.merge!(stream_specification(dynamo_db_table)) + attributes.merge!(tags_of(dynamo_db_table)) + attributes.merge!(ttl_of(dynamo_db_table)) + + resources["aws_dynamodb_table.#{module_name_of(dynamo_db_table)}"] = { + "type" => "aws_dynamodb_table", + "primary" => { + "id" => dynamo_db_table.table_name, + "attributes" => attributes, + "meta" => { + "schema_version" => "1" + } + } + } + resources + end + end + + private + + def tables + tables = [] + dynamo_db_tables.each do |table| + attributes = @client.describe_table({ + table_name: table + }).table + tables << attributes + end + return tables + end + + def attribute_definitions(dynamo_db_table) + attributes = { "attribute.#" => dynamo_db_table["attribute_definitions"].length.to_s} + dynamo_db_table["attribute_definitions"].each do |attr_defn| + attributes.merge!(attributes_definitions_of(attr_defn)) + end + attributes + end + + def attributes_definitions_of(attr_defn) + hashcode = attribute_hashcode(attr_defn) + attributes = { + "attribute.#{hashcode}.name" => attr_defn.attribute_name, + "attribute.#{hashcode}.type" => attr_defn.attribute_type, + } + attributes + end + + def attribute_hashcode(attr_defn) + hashcode = Zlib.crc32(attr_defn.attribute_name+"-") + end + + def global_indexes(dynamo_db_table) + attributes = {} + if dynamo_db_table["global_secondary_indexes"] + attributes = { "global_secondary_index.#" => dynamo_db_table["global_secondary_indexes"].length.to_s} + dynamo_db_table["global_secondary_indexes"].each do |global_sec_index| + attributes.merge!(global_secondary_indexes_of(global_sec_index)) + end + end + return attributes + end + + + def global_secondary_indexes_of(global_sec_index) + attributes = global_indexes_of(global_sec_index).merge!(global_index_non_key_attributes(global_sec_index)) + end + + def global_indexes_of(global_sec_index) + hashcode = global_index_hashcode(global_sec_index) + attributes = { + "global_secondary_index.#{hashcode}.hash_key" => find_key(global_sec_index,"HASH"), + "global_secondary_index.#{hashcode}.name" => global_sec_index.index_name, + "global_secondary_index.#{hashcode}.projection_type" => global_sec_index.projection.projection_type, + "global_secondary_index.#{hashcode}.range_key" => find_key(global_sec_index,"RANGE"), + "global_secondary_index.#{hashcode}.read_capacity" => global_sec_index.provisioned_throughput.read_capacity_units.to_s , + "global_secondary_index.#{hashcode}.write_capacity" => global_sec_index.provisioned_throughput.write_capacity_units.to_s, + } + attributes + end + + def find_key(index,key_type) + index["key_schema"].each do |schema| + if schema.key_type == key_type + return schema.attribute_name + else + return "" + end + end + end + + def global_index_non_key_attributes(global_sec_index) + attributes = {} + if !global_sec_index["projection"]["non_key_attributes"].nil? + hashcode = global_index_hashcode(global_sec_index) + attributes = {"global_secondary_index.#{hashcode}.non_key_attributes.#" => global_sec_index["projection"]["non_key_attributes"].length.to_s} + (0..global_sec_index["projection"]["non_key_attributes"].length.to_i-1).each do |index| + attributes.merge!({"global_secondary_index.#{hashcode}.non_key_attributes.#{index}" => attributes[index]}) + end + end + attributes + end + + + def global_index_hashcode(global_sec_index) + Zlib.crc32(global_sec_index["index_name"]+"-") + end + + def local_indexes(dynamo_db_table) + attributes = {} + if dynamo_db_table["local_secondary_indexes"] + attributes = {"local_secondary_index.#" => dynamo_db_table["local_secondary_indexes"].length.to_s} + dynamo_db_table["local_secondary_indexes"].each do |local_sec_index| + attributes.merge!(local_secondary_indexes_of(local_sec_index)) + end + end + return attributes + end + + def local_secondary_indexes_of(local_sec_index) + attributes = {} + hashcode = local_index_hashcode(local_sec_index) + attributes.merge!("local_secondary_index.#{hashcode}.range_key" => find_key(local_sec_index,"RANGE")) if !find_key(local_sec_index,"RANGE").empty? + attributes.merge!({ + "local_secondary_index.#{hashcode}.name" => local_sec_index.index_name, + "local_secondary_index.#{hashcode}.projection_type" => local_sec_index.projection.projection_type, + }) + attributes.merge!(local_index_non_key_attributes(local_sec_index)) + attributes + end + + def local_index_non_key_attributes(local_sec_index) + attributes = {} + if !local_sec_index["projection"]["non_key_attributes"].nil? + hashcode = local_index_hashcode(local_sec_index) + attributes = {"local_secondary_index.#{hashcode}.non_key_attributes.#" => local_sec_index["projection"]["non_key_attributes"].length.to_s} + (0..local_sec_index["projection"]["non_key_attributes"].length.to_i-1).each do |index| + attributes.merge!({"local_secondary_index.#{hashcode}.non_key_attributes.#{index}" => attributes[index]}) + end + end + attributes + end + + def local_index_hashcode(local_index) + Zlib.crc32(local_index["index_name"]+"-") + end + + def key_schema(dynamo_db_table) + attributes = {} + if dynamo_db_table["key_schema"] + attributes = {"key_schema.#" => dynamo_db_table["key_schema"].length.to_s} + if !find_key(dynamo_db_table,"HASH").empty? + attributes.merge!({"hash_key" => find_key(dynamo_db_table,"HASH")}) + end + if !find_key(dynamo_db_table,"RANGE").empty? + attributes.merge!({"range_key" => find_key(dynamo_db_table,"RANGE")}) + end + end + attributes + end + + def point_in_time_summary(dynamo_db_table) + resp = @client.describe_continuous_backups({ + table_name: dynamo_db_table["table_name"] + }) + if resp.continuous_backups_description.point_in_time_recovery_description.point_in_time_recovery_status == "ENABLED" + attributes = {"point_in_time_recovery.#" => 1.to_s} + attributes.merge!({"point_in_time_recovery.0.enabled" => true.to_s}) + else + attributes = {"point_in_time_recovery.#" => 0.to_s} + end + end + + def sse_description(dynamo_db_table) + attributes = {} + if dynamo_db_table.sse_description + if dynamo_db_table.sse_description.status == "ENABLED" + attributes = {"server_side_encryption.#" => 1} + attributes.merge!({"server_side_encryption.0.enabled" => true.to_s}) + else + attributes = {"server_side_encryption.#" => 0} + end + end + attributes + end + + def stream_specification(dynamo_db_table) + attributes = {} + if dynamo_db_table.stream_specification + attributes = {"stream_view_type" => dynamo_db_table.stream_specification.stream_view_type} if dynamo_db_table.stream_specification.stream_enabled + end + attributes + end + + def ttl_of(dynamo_db_table) + attributes = {} + if ttl_values(dynamo_db_table) + ttl = ttl_values(dynamo_db_table) + hashcode = ttl_hashcode(ttl.attribute_name) + attributes = {"ttl.#" => 1.to_s} if + attributes["ttl.#{hashcode}.attribute_name"] = ttl.attribute_name + attributes["ttl.#{hashcode}.enabled"] = true.to_s + end + return attributes + end + + def ttl_hashcode(attribute) + Zlib.crc32(attribute) + end + + def tags_of(dynamo_db_table) + attributes = {} + if tags(dynamo_db_table) + tags = tags(dynamo_db_table) + attributes = { "tags.%" => tags.length.to_s } + tags.each do |tag| + attributes["tags.#{tag.key}"] = tag.value + end + end + attributes + end + + def dynamo_db_tables + a = @client.list_tables.map(&:table_names).flatten + end + + def ttl_values(dynamo_db_table) + ttl = @client.describe_time_to_live({ + table_name: dynamo_db_table.table_name + }).time_to_live_description + if ttl.time_to_live_status == "ENABLED" + return ttl + else + return nil + end + end + + def tags(dynamo_db_table) + return tags if !@client.list_tags_of_resource({resource_arn: dynamo_db_table.table_arn}).tags.empty? + end + + def module_name_of(dynamo_db_table) + normalize_module_name(dynamo_db_table['table_name']) + end + end + end +end diff --git a/lib/terraforming/template/tf/dynamo_db.erb b/lib/terraforming/template/tf/dynamo_db.erb new file mode 100644 index 00000000..120b09a3 --- /dev/null +++ b/lib/terraforming/template/tf/dynamo_db.erb @@ -0,0 +1,64 @@ +<%- tables.each do |table| -%> +resource "aws_dynamodb_table" "<%= table.table_name -%>" { + name = "<%= table.table_name -%>" + read_capacity = <%= table.provisioned_throughput.read_capacity_units %> + write_capacity = <%= table.provisioned_throughput.write_capacity_units %> +<%- table.key_schema.each do |key| -%> + <%= key.key_type.downcase -%>_key = <%= key.attribute_name.inspect %> +<%- end %> +<%- table.attribute_definitions.each do |attribute| -%> + attribute { + name = "<%= attribute.attribute_name -%>" + type = "<%= attribute.attribute_type -%>" + } +<%- end -%> +<%- if ttl_values(table) -%> + ttl { + attribute_name = <%= ttl_values(table).attribute_name -%> + enabled = true + } +<%- end -%> + +<%- if tags(table) -%> + tags { + <%- tags(table).each do |tag| -%> + <%= tag.key -%> = "<%= tag.value -%>" + <%- end -%> + } +<%- end -%> +<%- Array(table.global_secondary_indexes).each do |index| -%> + global_secondary_index { + name = "<%= index.index_name -%>" + <%- index.key_schema.each do |key| -%> + <%= key.key_type.downcase -%>_key = "<%= key.attribute_name -%>" + <%- end -%> + read_capacity = <%= index.provisioned_throughput.read_capacity_units -%> + write_capacity = <%= index.provisioned_throughput.write_capacity_units -%> + projection_type = "<%= index.projection.projection_type -%>" + <%- keys = index.projection.non_key_attributes -%> + <%- if Array(keys).size > 0 -%> + non_key_attributes = <%= keys.inspect -%> + <%- end -%> + } +<%- end -%> +<%- Array(table.local_secondary_indexes).each do |index| -%> + local_secondary_index { + name = "<%= index.index_name -%>" + <%- index.key_schema.each do |key| -%> + <%- if key.key_type.downcase == "range" -%> + <%= key.key_type.downcase -%>_key = "<%= key.attribute_name -%>" + <%- end -%> + <%- end -%> + projection_type = "<%= index.projection.projection_type -%>" + <%- keys = index.projection.non_key_attributes -%> + <%- if Array(keys).size > 0 -%> + non_key_attributes = <%= keys.inspect -%> + <%- end -%> + } +<%- end -%> +<%- if table.stream_specification -%> + stream_enabled = <%= table.stream_specification.stream_enabled -%> + stream_view_type = <%= table.stream_specification.stream_view_type.inspect-%> +<%- end -%> +} +<%- end -%> \ No newline at end of file diff --git a/spec/lib/terraforming/cli_spec.rb b/spec/lib/terraforming/cli_spec.rb index c90f4f9f..fe118f04 100644 --- a/spec/lib/terraforming/cli_spec.rb +++ b/spec/lib/terraforming/cli_spec.rb @@ -85,6 +85,13 @@ module Terraforming it_behaves_like "CLI examples" end + describe "ddb" do + let(:klass) { Terraforming::Resource::DynamoDb } + let(:command) { :ddb } + + it_behaves_like "CLI examples" + end + describe "ec2" do let(:klass) { Terraforming::Resource::EC2 } let(:command) { :ec2 } diff --git a/spec/lib/terraforming/resource/dynamo_db_spec.rb b/spec/lib/terraforming/resource/dynamo_db_spec.rb new file mode 100644 index 00000000..32228f22 --- /dev/null +++ b/spec/lib/terraforming/resource/dynamo_db_spec.rb @@ -0,0 +1,145 @@ +require "spec_helper" + +module Terraforming + module Resource + describe DynamoDb do + let(:client) do + Aws::DynamoDB::Client.new(stub_responses: true) + end + + let(:tables) do + [ + "test-ddb" + ] + end + + let(:test_dynamodb_table) do + { + attribute_definitions: + [ + { attribute_name: "account_id", attribute_type: "N" }, + { attribute_name: "action_timestamp", attribute_type: "N" }, + { attribute_name: "type_parentid_timestamp", attribute_type: "S" } + ], + table_name: "test-ddb", + key_schema: [ + {attribute_name: "account_id", key_type: "HASH"}, + {attribute_name: "type_parentid_timestamp", key_type: "RANGE"} + ], + table_status: "ACTIVE", + creation_date_time: Time.parse("2016-08-31 06:23:57 UTC"), + provisioned_throughput: { number_of_decreases_today: 0, read_capacity_units: 1, write_capacity_units: 1 }, + table_size_bytes: 0, + item_count: 0, + table_arn: "arn:aws:dynamodb:eu-central-1:123456789:table/test-ddb", + local_secondary_indexes: [ + { + index_name: "action_timestamp_index", + key_schema: [ + {attribute_name: "account_id", key_type: "HASH"}, + {attribute_name: "action_timestamp", key_type: "RANGE"} + ], + projection: { projection_type: "ALL" }, + index_size_bytes: 0, + item_count: 0, + index_arn: "arn:aws:dynamodb:eu-central-1:123456789:table/test-ddb/index/action_timestamp_index"} + ] + } + end + + let(:test_ddb_continuous_backups_description) do + { + continuous_backups_status: "ENABLED", + point_in_time_recovery_description: {point_in_time_recovery_status: "DISABLED"} + } + end + + let(:test_ddb_describe_time_to_live) do + {time_to_live_status: "DISABLED"} + end + + let(:test_ddb_tags) do + [] + end + + before do + client.stub_responses(:list_tables, table_names: tables) + client.stub_responses(:describe_table, table: test_dynamodb_table) + client.stub_responses(:describe_continuous_backups, continuous_backups_description: test_ddb_continuous_backups_description) + client.stub_responses(:describe_time_to_live, time_to_live_description: test_ddb_describe_time_to_live ) + client.stub_responses(:list_tags_of_resource, tags: test_ddb_tags) + end + + describe ".tf" do + it "should generate tf" do + expect(described_class.tf(client: client)).to eq <<-EOS +resource "aws_dynamodb_table" "test-ddb" { + name = "test-ddb" + read_capacity = 1 + write_capacity = 1 + hash_key = "account_id" + range_key = "type_parentid_timestamp" + + attribute { + name = "account_id" + type = "N" + } + attribute { + name = "action_timestamp" + type = "N" + } + attribute { + name = "type_parentid_timestamp" + type = "S" + } + + local_secondary_index { + name = "action_timestamp_index" + range_key = "action_timestamp" + projection_type = "ALL" + } +} + EOS + end + end + + describe ".tfstate" do + it "should generate tfstate" do + expect(described_class.tfstate(client: client)).to eq({ + "aws_dynamodb_table.test-ddb" => { + "type" => "aws_dynamodb_table", + "primary" => { + "id" => "test-ddb", + "attributes" => { + "arn" => "arn:aws:dynamodb:eu-central-1:123456789:table/test-ddb", + "id" => "test-ddb", + "name" => "test-ddb", + "read_capacity" => "1", + "stream_arn" => "", + "stream_label" => "", + "write_capacity" => "1", + "attribute.#" => "3", + "attribute.3170009653.name" => "account_id", + "attribute.3170009653.type" => "N", + "attribute.901452415.name" => "action_timestamp", + "attribute.901452415.type" => "N", + "attribute.2131915850.name" => "type_parentid_timestamp", + "attribute.2131915850.type" => "S", + "local_secondary_index.#" => "1", + "local_secondary_index.2469045277.name" => "action_timestamp_index", + "local_secondary_index.2469045277.projection_type" => "ALL", + "key_schema.#" => "2", + "hash_key" => "account_id", + "point_in_time_recovery.#" => "0" + }, + "meta" => { + "schema_version" => "1" + } + } + } + }) + end + end + end + end +end diff --git a/terraforming.gemspec b/terraforming.gemspec index b0470232..96c7adf2 100644 --- a/terraforming.gemspec +++ b/terraforming.gemspec @@ -21,6 +21,7 @@ Gem::Specification.new do |spec| spec.add_dependency "aws-sdk-autoscaling", "~> 1" spec.add_dependency "aws-sdk-cloudwatch", "~> 1" + spec.add_dependency "aws-sdk-dynamodb", "~> 1.18" spec.add_dependency "aws-sdk-ec2", "~> 1" spec.add_dependency "aws-sdk-efs", "~> 1" spec.add_dependency "aws-sdk-elasticache", "~> 1" From eb0c5c49a89a9ffb725425370f494d81a735156e Mon Sep 17 00:00:00 2001 From: laxmiprasanna-gunna Date: Thu, 13 Dec 2018 12:13:32 +0530 Subject: [PATCH 2/3] Fix ttl attributes --- lib/terraforming/template/tf/dynamo_db.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/terraforming/template/tf/dynamo_db.erb b/lib/terraforming/template/tf/dynamo_db.erb index 120b09a3..19565db8 100644 --- a/lib/terraforming/template/tf/dynamo_db.erb +++ b/lib/terraforming/template/tf/dynamo_db.erb @@ -14,7 +14,7 @@ resource "aws_dynamodb_table" "<%= table.table_name -%>" { <%- end -%> <%- if ttl_values(table) -%> ttl { - attribute_name = <%= ttl_values(table).attribute_name -%> + attribute_name = <%= ttl_values(table).attribute_name.inspect %> enabled = true } <%- end -%> From f9cf9888aafd04e2c0dca35b90204a9166233bf6 Mon Sep 17 00:00:00 2001 From: laxmiprasanna-gunna Date: Tue, 18 Dec 2018 10:21:24 +0530 Subject: [PATCH 3/3] Add more test cases and fix template spacing --- lib/terraforming/resource/dynamo_db.rb | 35 ++- lib/terraforming/template/tf/dynamo_db.erb | 36 +-- .../terraforming/resource/dynamo_db_spec.rb | 266 ++++++++++++++---- 3 files changed, 254 insertions(+), 83 deletions(-) diff --git a/lib/terraforming/resource/dynamo_db.rb b/lib/terraforming/resource/dynamo_db.rb index a8a49c1f..9230a399 100644 --- a/lib/terraforming/resource/dynamo_db.rb +++ b/lib/terraforming/resource/dynamo_db.rb @@ -133,7 +133,7 @@ def global_index_non_key_attributes(global_sec_index) hashcode = global_index_hashcode(global_sec_index) attributes = {"global_secondary_index.#{hashcode}.non_key_attributes.#" => global_sec_index["projection"]["non_key_attributes"].length.to_s} (0..global_sec_index["projection"]["non_key_attributes"].length.to_i-1).each do |index| - attributes.merge!({"global_secondary_index.#{hashcode}.non_key_attributes.#{index}" => attributes[index]}) + attributes.merge!({"global_secondary_index.#{hashcode}.non_key_attributes.#{index}" => global_sec_index["projection"]["non_key_attributes"][index]}) end end attributes @@ -173,7 +173,7 @@ def local_index_non_key_attributes(local_sec_index) hashcode = local_index_hashcode(local_sec_index) attributes = {"local_secondary_index.#{hashcode}.non_key_attributes.#" => local_sec_index["projection"]["non_key_attributes"].length.to_s} (0..local_sec_index["projection"]["non_key_attributes"].length.to_i-1).each do |index| - attributes.merge!({"local_secondary_index.#{hashcode}.non_key_attributes.#{index}" => attributes[index]}) + attributes.merge!({"local_secondary_index.#{hashcode}.non_key_attributes.#{index}" => local_sec_index["projection"]["non_key_attributes"][index]}) end end attributes @@ -190,9 +190,6 @@ def key_schema(dynamo_db_table) if !find_key(dynamo_db_table,"HASH").empty? attributes.merge!({"hash_key" => find_key(dynamo_db_table,"HASH")}) end - if !find_key(dynamo_db_table,"RANGE").empty? - attributes.merge!({"range_key" => find_key(dynamo_db_table,"RANGE")}) - end end attributes end @@ -213,11 +210,11 @@ def sse_description(dynamo_db_table) attributes = {} if dynamo_db_table.sse_description if dynamo_db_table.sse_description.status == "ENABLED" - attributes = {"server_side_encryption.#" => 1} + attributes = {"server_side_encryption.#" => 1.to_s} attributes.merge!({"server_side_encryption.0.enabled" => true.to_s}) - else - attributes = {"server_side_encryption.#" => 0} end + else + attributes.merge!({"server_side_encryption.#" => 0.to_s}) end attributes end @@ -232,11 +229,11 @@ def stream_specification(dynamo_db_table) def ttl_of(dynamo_db_table) attributes = {} - if ttl_values(dynamo_db_table) - ttl = ttl_values(dynamo_db_table) - hashcode = ttl_hashcode(ttl.attribute_name) - attributes = {"ttl.#" => 1.to_s} if - attributes["ttl.#{hashcode}.attribute_name"] = ttl.attribute_name + ttl = ttl_values(dynamo_db_table) + if !ttl.empty? + hashcode = ttl_hashcode(ttl.first) + attributes = {"ttl.#" => 1.to_s} + attributes["ttl.#{hashcode}.attribute_name"] = ttl.first attributes["ttl.#{hashcode}.enabled"] = true.to_s end return attributes @@ -248,8 +245,8 @@ def ttl_hashcode(attribute) def tags_of(dynamo_db_table) attributes = {} - if tags(dynamo_db_table) - tags = tags(dynamo_db_table) + tags = tags(dynamo_db_table) + if !tags.empty? attributes = { "tags.%" => tags.length.to_s } tags.each do |tag| attributes["tags.#{tag.key}"] = tag.value @@ -267,14 +264,14 @@ def ttl_values(dynamo_db_table) table_name: dynamo_db_table.table_name }).time_to_live_description if ttl.time_to_live_status == "ENABLED" - return ttl - else - return nil + return [ttl.attribute_name] + else + return [] end end def tags(dynamo_db_table) - return tags if !@client.list_tags_of_resource({resource_arn: dynamo_db_table.table_arn}).tags.empty? + resp = @client.list_tags_of_resource({resource_arn: dynamo_db_table.table_arn}).tags end def module_name_of(dynamo_db_table) diff --git a/lib/terraforming/template/tf/dynamo_db.erb b/lib/terraforming/template/tf/dynamo_db.erb index 19565db8..f7922578 100644 --- a/lib/terraforming/template/tf/dynamo_db.erb +++ b/lib/terraforming/template/tf/dynamo_db.erb @@ -12,33 +12,25 @@ resource "aws_dynamodb_table" "<%= table.table_name -%>" { type = "<%= attribute.attribute_type -%>" } <%- end -%> -<%- if ttl_values(table) -%> +<%- ttl_values(table).each do |attr| -%> ttl { - attribute_name = <%= ttl_values(table).attribute_name.inspect %> + attribute_name = <%= attr.inspect %> enabled = true } <%- end -%> - -<%- if tags(table) -%> - tags { - <%- tags(table).each do |tag| -%> - <%= tag.key -%> = "<%= tag.value -%>" - <%- end -%> - } -<%- end -%> <%- Array(table.global_secondary_indexes).each do |index| -%> global_secondary_index { name = "<%= index.index_name -%>" <%- index.key_schema.each do |key| -%> <%= key.key_type.downcase -%>_key = "<%= key.attribute_name -%>" <%- end -%> - read_capacity = <%= index.provisioned_throughput.read_capacity_units -%> - write_capacity = <%= index.provisioned_throughput.write_capacity_units -%> - projection_type = "<%= index.projection.projection_type -%>" + read_capacity = <%= index.provisioned_throughput.read_capacity_units %> + write_capacity = <%= index.provisioned_throughput.write_capacity_units %> + projection_type = "<%= index.projection.projection_type %>" <%- keys = index.projection.non_key_attributes -%> <%- if Array(keys).size > 0 -%> non_key_attributes = <%= keys.inspect -%> - <%- end -%> + <%- end %> } <%- end -%> <%- Array(table.local_secondary_indexes).each do |index| -%> @@ -53,12 +45,22 @@ resource "aws_dynamodb_table" "<%= table.table_name -%>" { <%- keys = index.projection.non_key_attributes -%> <%- if Array(keys).size > 0 -%> non_key_attributes = <%= keys.inspect -%> - <%- end -%> + <%- end %> + } +<%- end -%> +<%- tags(table).each do |tag| -%> + tags { + <%= tag.key %> = "<%= tag.value -%>" } <%- end -%> <%- if table.stream_specification -%> - stream_enabled = <%= table.stream_specification.stream_enabled -%> - stream_view_type = <%= table.stream_specification.stream_view_type.inspect-%> + stream_enabled = <%= table.stream_specification.stream_enabled %> + stream_view_type = <%= table.stream_specification.stream_view_type.inspect %> +<%- end -%> +<%- if table.sse_description -%> + server_side_encryption { + enabled = true + } <%- end -%> } <%- end -%> \ No newline at end of file diff --git a/spec/lib/terraforming/resource/dynamo_db_spec.rb b/spec/lib/terraforming/resource/dynamo_db_spec.rb index 32228f22..7ec706e8 100644 --- a/spec/lib/terraforming/resource/dynamo_db_spec.rb +++ b/spec/lib/terraforming/resource/dynamo_db_spec.rb @@ -9,17 +9,19 @@ module Resource let(:tables) do [ - "test-ddb" + "test-ddb","new-ddb" ] end - let(:test_dynamodb_table) do + let(:test_ddb_table) do { attribute_definitions: [ { attribute_name: "account_id", attribute_type: "N" }, { attribute_name: "action_timestamp", attribute_type: "N" }, - { attribute_name: "type_parentid_timestamp", attribute_type: "S" } + { attribute_name: "type_parentid_timestamp", attribute_type: "S" }, + {attribute_name: "newky", attribute_type: "S"}, + {attribute_name: "newsortkey", attribute_type: "S"}, ], table_name: "test-ddb", key_schema: [ @@ -39,35 +41,107 @@ module Resource {attribute_name: "account_id", key_type: "HASH"}, {attribute_name: "action_timestamp", key_type: "RANGE"} ], - projection: { projection_type: "ALL" }, + projection: { projection_type: "INCLUDE", non_key_attributes: ["fghi", "jklm"] }, index_size_bytes: 0, item_count: 0, index_arn: "arn:aws:dynamodb:eu-central-1:123456789:table/test-ddb/index/action_timestamp_index"} - ] + ], + global_secondary_indexes: [ + { + index_name: "newky-newsortkey-index", + key_schema: [ + {attribute_name: "newky", key_type: "HASH"}, + {attribute_name: "newsortkey", key_type: "RANGE"} + ], + projection: { projection_type: "INCLUDE", non_key_attributes: ["abcd", "efgh"] }, + index_status: "ACTIVE", + provisioned_throughput: { number_of_decreases_today: 0, read_capacity_units: 1, write_capacity_units: 1 }, + index_size_bytes: 0, + item_count: 0, + index_arn: "arn:aws:dynamodb:eu-central-1:123456789:table/test-ddb/index/newky-newsortkey-index"} + ], + stream_specification: {stream_enabled: true, stream_view_type: "NEW_AND_OLD_IMAGES"}, + latest_stream_label: Time.parse("2016-08-31 06:23:57 UTC").to_s, + latest_stream_arn: "arn:aws:dynamodb:eu-central-1:123456789:table/test-ddb/stream/"+Time.parse("2016-08-31 06:23:57 UTC").to_s, + sse_description: { + status: "ENABLED" + } + } + end + + let(:new_ddb_table) do + { + attribute_definitions:[ + {:attribute_name=>"id", :attribute_type=>"S"}, + {:attribute_name=>"time", :attribute_type=>"N"} + ], + table_name: "new-ddb", + key_schema: [ + {:attribute_name=>"id", :key_type=>"HASH"}, + {:attribute_name=>"time", :key_type=>"RANGE"} + ], + table_status: "ACTIVE", + creation_date_time: Time.parse("2016-08-31 06:23:57 UTC"), + provisioned_throughput: {number_of_decreases_today: 0, read_capacity_units: 5, write_capacity_units: 5}, + table_size_bytes: 12345, + item_count: 11222, + :table_arn=>"arn:aws:dynamodb:eu-central-1:123456789:table/new-ddb", + :table_id=>"new-ddb" } end let(:test_ddb_continuous_backups_description) do { - continuous_backups_status: "ENABLED", - point_in_time_recovery_description: {point_in_time_recovery_status: "DISABLED"} + continuous_backups_status: "ENABLED", + point_in_time_recovery_description: { + point_in_time_recovery_status: "ENABLED" + } + } + end + + let(:new_ddb_continuous_backups_description) do + { + continuous_backups_status: "ENABLED", + point_in_time_recovery_description: { + point_in_time_recovery_status: "DISABLED" + } } end let(:test_ddb_describe_time_to_live) do - {time_to_live_status: "DISABLED"} + {:time_to_live_status=>"ENABLED", :attribute_name=>"1"} + end + + let(:new_ddb_describe_time_to_live) do + {:time_to_live_status=>"DISABLED"} end let(:test_ddb_tags) do + [{:key=>"abcd", :value=>"efgh"}] + end + + let(:new_ddb_tags) do [] end before do client.stub_responses(:list_tables, table_names: tables) - client.stub_responses(:describe_table, table: test_dynamodb_table) - client.stub_responses(:describe_continuous_backups, continuous_backups_description: test_ddb_continuous_backups_description) - client.stub_responses(:describe_time_to_live, time_to_live_description: test_ddb_describe_time_to_live ) - client.stub_responses(:list_tags_of_resource, tags: test_ddb_tags) + client.stub_responses(:describe_table, [ + {table: test_ddb_table}, + {table: new_ddb_table} + ]) + client.stub_responses(:describe_continuous_backups,[ + {continuous_backups_description: test_ddb_continuous_backups_description}, + {continuous_backups_description: new_ddb_continuous_backups_description} + ]) + client.stub_responses(:describe_time_to_live, [ + {time_to_live_description: test_ddb_describe_time_to_live}, + {time_to_live_description: new_ddb_describe_time_to_live} + ]) + client.stub_responses(:list_tags_of_resource, [ + {tags: test_ddb_tags}, + {tags: new_ddb_tags} + ]) end describe ".tf" do @@ -92,54 +166,152 @@ module Resource name = "type_parentid_timestamp" type = "S" } - + attribute { + name = "newky" + type = "S" + } + attribute { + name = "newsortkey" + type = "S" + } + ttl { + attribute_name = "1" + enabled = true + } + global_secondary_index { + name = "newky-newsortkey-index" + hash_key = "newky" + range_key = "newsortkey" + read_capacity = 1 + write_capacity = 1 + projection_type = "INCLUDE" + non_key_attributes = ["abcd", "efgh"] + } local_secondary_index { name = "action_timestamp_index" range_key = "action_timestamp" - projection_type = "ALL" + projection_type = "INCLUDE" + non_key_attributes = ["fghi", "jklm"] + } + tags { + abcd = "efgh" + } + stream_enabled = true + stream_view_type = "NEW_AND_OLD_IMAGES" + server_side_encryption { + enabled = true } } - EOS +resource "aws_dynamodb_table" "new-ddb" { + name = "new-ddb" + read_capacity = 5 + write_capacity = 5 + hash_key = "id" + range_key = "time" + + attribute { + name = "id" + type = "S" + } + attribute { + name = "time" + type = "N" + } +} +EOS end end describe ".tfstate" do it "should generate tfstate" do expect(described_class.tfstate(client: client)).to eq({ - "aws_dynamodb_table.test-ddb" => { - "type" => "aws_dynamodb_table", - "primary" => { - "id" => "test-ddb", - "attributes" => { - "arn" => "arn:aws:dynamodb:eu-central-1:123456789:table/test-ddb", - "id" => "test-ddb", - "name" => "test-ddb", - "read_capacity" => "1", - "stream_arn" => "", - "stream_label" => "", - "write_capacity" => "1", - "attribute.#" => "3", - "attribute.3170009653.name" => "account_id", - "attribute.3170009653.type" => "N", - "attribute.901452415.name" => "action_timestamp", - "attribute.901452415.type" => "N", - "attribute.2131915850.name" => "type_parentid_timestamp", - "attribute.2131915850.type" => "S", - "local_secondary_index.#" => "1", - "local_secondary_index.2469045277.name" => "action_timestamp_index", - "local_secondary_index.2469045277.projection_type" => "ALL", - "key_schema.#" => "2", - "hash_key" => "account_id", - "point_in_time_recovery.#" => "0" + "aws_dynamodb_table.test-ddb"=> + { + "type"=>"aws_dynamodb_table", + "primary"=> + { + "id"=>"test-ddb", + "attributes"=> + { + "arn"=>"arn:aws:dynamodb:eu-central-1:123456789:table/test-ddb", + "id"=>"test-ddb", + "name"=>"test-ddb", + "read_capacity"=>"1", + "stream_arn"=>"arn:aws:dynamodb:eu-central-1:123456789:table/test-ddb/stream/2016-08-31 06:23:57 UTC", + "stream_label"=>"2016-08-31 06:23:57 UTC", + "write_capacity"=>"1", + "attribute.#"=>"5", + "attribute.3170009653.name"=>"account_id", + "attribute.3170009653.type"=>"N", + "attribute.901452415.name"=>"action_timestamp", + "attribute.901452415.type"=>"N", + "attribute.2131915850.name"=>"type_parentid_timestamp", + "attribute.2131915850.type"=>"S", + "attribute.3685094810.name"=>"newky", + "attribute.3685094810.type"=>"S", + "attribute.3333016131.name"=>"newsortkey", + "attribute.3333016131.type"=>"S", + "global_secondary_index.#"=>"1", + "global_secondary_index.1661317069.hash_key"=>"newky", + "global_secondary_index.1661317069.name"=>"newky-newsortkey-index", + "global_secondary_index.1661317069.projection_type"=>"INCLUDE", + "global_secondary_index.1661317069.range_key"=>"", + "global_secondary_index.1661317069.read_capacity"=>"1", + "global_secondary_index.1661317069.write_capacity"=>"1", + "global_secondary_index.1661317069.non_key_attributes.#"=>"2", + "global_secondary_index.1661317069.non_key_attributes.0"=>"abcd", + "global_secondary_index.1661317069.non_key_attributes.1"=>"efgh", + "local_secondary_index.#"=>"1", + "local_secondary_index.2469045277.name"=>"action_timestamp_index", + "local_secondary_index.2469045277.projection_type"=>"INCLUDE", + "local_secondary_index.2469045277.non_key_attributes.#"=>"2", + "local_secondary_index.2469045277.non_key_attributes.0"=>"fghi", + "local_secondary_index.2469045277.non_key_attributes.1"=>"jklm", + "key_schema.#"=>"2", + "hash_key"=>"account_id", + "point_in_time_recovery.#"=>"1", + "point_in_time_recovery.0.enabled"=>"true", + "server_side_encryption.#"=>"1", + "server_side_encryption.0.enabled"=>"true", + "stream_view_type"=>"NEW_AND_OLD_IMAGES", + "tags.%"=>"1", + "tags.abcd"=>"efgh", + "ttl.#"=>"1", + "ttl.2212294583.attribute_name"=>"1", + "ttl.2212294583.enabled"=>"true" }, - "meta" => { - "schema_version" => "1" + "meta"=>{"schema_version"=>"1"} + } + }, + "aws_dynamodb_table.new-ddb"=> + { + "type"=>"aws_dynamodb_table", + "primary"=> + { + "id"=>"new-ddb", + "attributes"=> + { + "arn"=>"arn:aws:dynamodb:eu-central-1:123456789:table/new-ddb", + "id"=>"new-ddb", + "name"=>"new-ddb", + "read_capacity"=>"5", + "stream_arn"=>"", + "stream_label"=>"", + "write_capacity"=>"5", + "attribute.#"=>"2", + "attribute.4228504427.name"=>"id", + "attribute.4228504427.type"=>"S", + "attribute.2432995967.name"=>"time", + "attribute.2432995967.type"=>"N", + "key_schema.#"=>"2", "hash_key"=>"id", + "point_in_time_recovery.#"=>"0", + "server_side_encryption.#"=>"0"}, + "meta"=>{"schema_version"=>"1"} + } } - } - } - }) - end - end + }) + end end end + end end