RDS instance: Terraform insisting on destroy-recreate RDS instances although their IDs are unchanged

[dohoangkhiem]

We inspected our plan and realized Terraform keeps wanting to replace our RDS instances although their name, AZ have not changed, see below plan

-/+ module.magnolia.aws_db_instance.public_magnolia[0] (new resource required)
      id:                                "group1-site1-live-5-5-6-public1" => <computed> (forces new resource)
      address:                           "group1-site1-live-5-5-6-public1.czwmzj5tmahw.ap-southeast-1.rds.amazonaws.com" => <computed>
      allocated_storage:                 "5" => "5"
      apply_immediately:                 "" => <computed>
      arn:                               "arn:aws:rds:ap-southeast-1:218832052474:db:group1-site1-live-5-5-6-public1" => <computed>
      auto_minor_version_upgrade:        "true" => "true"
      availability_zone:                 "ap-southeast-1a" => <computed> (forces new resource)
      backup_retention_period:           "0" => <computed>
      backup_window:                     "20:55-21:25" => <computed>
      ca_cert_identifier:                "rds-ca-2015" => <computed>
      character_set_name:                "" => <computed>
      copy_tags_to_snapshot:             "true" => "true"
      db_subnet_group_name:              "khiem-db-subnet" => "khiem-db-subnet"
      endpoint:                          "group1-site1-live-5-5-6-public1.czwmzj5tmahw.ap-southeast-1.rds.amazonaws.com:5432" => <computed>
      engine:                            "postgres" => "postgres"
      engine_version:                    "9.5.4" => "9.5.4"
      hosted_zone_id:                    "Z2G0U3KFCY8NZ5" => <computed>
      identifier:                        "group1-site1-live-5-5-6-public1" => "group1-site1-live-5-5-6-public1"
      identifier_prefix:                 "" => <computed>
      instance_class:                    "db.t2.micro" => "db.t2.micro"
      kms_key_id:                        "" => <computed>
      license_model:                     "postgresql-license" => <computed>
      maintenance_window:                "fri:17:06-fri:17:36" => <computed>
      monitoring_interval:               "0" => "0"
      monitoring_role_arn:               "" => <computed>
      multi_az:                          "false" => "false"
      name:                              "magnolia" => "magnolia"
      option_group_name:                 "default:postgres-9-5" => <computed>
      parameter_group_name:              "default.postgres9.5" => "default.postgres9.5"
      password:                          <sensitive> => <sensitive> (attribute changed)
      port:                              "5432" => <computed>
      publicly_accessible:               "false" => "false"
      replicas.#:                        "0" => <computed>
      resource_id:                       "db-YHVKOYAXV7V3H33OEAH4SOKR6Q" => <computed>
      skip_final_snapshot:               "true" => "true"
      status:                            "available" => <computed>
      storage_type:                      "gp2" => "gp2"
      tags.%:                            "2" => "2"
      tags.Organization:                 "group1-site1" => "group1-site1"
      tags.Subscription:                 "group1-site1" => "group1-site1"
      timezone:                          "" => <computed>
      username:                          "postgres" => "postgres"
      vpc_security_group_ids.#:          "1" => "1"
      vpc_security_group_ids.1523498910: "sg-814006e7" => "sg-814006e7"

-/+ module.magnolia.aws_db_instance.public_magnolia[1] (new resource required)
      id:                                "group1-site1-live-5-5-6-public2" => <computed> (forces new resource)
      address:                           "group1-site1-live-5-5-6-public2.czwmzj5tmahw.ap-southeast-1.rds.amazonaws.com" => <computed>
      allocated_storage:                 "5" => "5"
      apply_immediately:                 "" => <computed>
      arn:                               "arn:aws:rds:ap-southeast-1:218832052474:db:group1-site1-live-5-5-6-public2" => <computed>
      auto_minor_version_upgrade:        "true" => "true"
      availability_zone:                 "ap-southeast-1a" => <computed> (forces new resource)
      backup_retention_period:           "0" => <computed>
      backup_window:                     "20:01-20:31" => <computed>
      ca_cert_identifier:                "rds-ca-2015" => <computed>
      character_set_name:                "" => <computed>
      copy_tags_to_snapshot:             "true" => "true"
      db_subnet_group_name:              "khiem-db-subnet" => "khiem-db-subnet"
      endpoint:                          "group1-site1-live-5-5-6-public2.czwmzj5tmahw.ap-southeast-1.rds.amazonaws.com:5432" => <computed>
      engine:                            "postgres" => "postgres"
      engine_version:                    "9.5.4" => "9.5.4"
      hosted_zone_id:                    "Z2G0U3KFCY8NZ5" => <computed>
      identifier:                        "group1-site1-live-5-5-6-public2" => "group1-site1-live-5-5-6-public2"
      identifier_prefix:                 "" => <computed>
      instance_class:                    "db.t2.micro" => "db.t2.micro"
      kms_key_id:                        "" => <computed>
      license_model:                     "postgresql-license" => <computed>
      maintenance_window:                "mon:18:18-mon:18:48" => <computed>
      monitoring_interval:               "0" => "0"
      monitoring_role_arn:               "" => <computed>
      multi_az:                          "false" => "false"
      name:                              "magnolia" => "magnolia"
      option_group_name:                 "default:postgres-9-5" => <computed>
      parameter_group_name:              "default.postgres9.5" => "default.postgres9.5"
      password:                          <sensitive> => <sensitive> (attribute changed)
      port:                              "5432" => <computed>
      publicly_accessible:               "false" => "false"
      replicas.#:                        "0" => <computed>
      resource_id:                       "db-AD5UTAPUZHDF44GPUTMR2FKOZQ" => <computed>
      skip_final_snapshot:               "true" => "true"
      status:                            "available" => <computed>
      storage_type:                      "gp2" => "gp2"
      tags.%:                            "2" => "2"
      tags.Organization:                 "group1-site1" => "group1-site1"
      tags.Subscription:                 "group1-site1" => "group1-site1"
      timezone:                          "" => <computed>
      username:                          "postgres" => "postgres"
      vpc_security_group_ids.#:          "1" => "1"
      vpc_security_group_ids.1523498910: "sg-814006e7" => "sg-814006e7"


Plan: 2 to add, 0 to change, 2 to destroy.

The line
identifier: "group1-site1-live-5-5-6-public1" => "group1-site1-live-5-5-6-public1"

shows that no ID changes, and if we apply this it turns out that even availability zone does not change. But Terraform plan reports

id: "group1-site1-live-5-5-6-public1" => <computed> (forces new resource)
that makes me confused about the difference between id and identifier?

How/why this plan could happen? We have another RDS instance which is almost the same as above but it's not in a list (so, count = 1) and the plan does not touch that instance (which is correct behaviour). Is this a bug?

There's a note that these existing RDS instances were imported to the state manually like following

terraform import module.magnolia.aws_db_instance.public_magnolia[0] group1-site1-live-5-5-7-public1

terraform import module.magnolia.aws_db_instance.public_magnolia[1] group1-site1-live-5-5-7-public2

Terraform version is 0.10.7, we tried with 0.10.8 and 0.11 and it's still the same plan.

[jbardin]

Hi @dohoangkhiem,

Can you share the config that you have which is causing this?
It does look like availability_zone is the culprit here, but we need to see how that value is being computed to know for sure.

[dohoangkhiem]

@jbardin

You're right! I also found that availability_zone is the culprit, that leads to another situation, before this we didn't set the availability_zone for our RDS instance, then now for supporting multiple AZs we have updated to set the availability_zone, in this case actually the availability_zone will be set to empty value "". We have an expression like this

availability_zone = "${(var.multi_az_deployment == "true")? var.az_list[count.index % length(var.az_list)] : ""}"

and here we have multi_az_deployment = "false"

So, seems Terraform really considers no availability_zone and setting availability_zone = "" are different, and it tried to recreate the instance. If I'm not wrong without the availability_zone set RDS instance are created with random AZ, how Terraform would handle AZ if availability_zone is set with empty value?

Do you think no availability_zone and availability_zone = "" should be handled the same way and Terraform shouldn't try to recreate RDS instances this case?

[jbardin]

@dohoangkhiem,

I have a feeling that the aws provider doesn't have a default value for availability_zone, so in that case I think unset and "" would show up as a diff.

However in this case, I still think the issue is that availability_zone is showing up as <computed>. This is likely because of the conditional expression you're using, but without a config showing all the references, I can't say for sure.

[dohoangkhiem]

@jbardin thanks for your response

Below is our configuration for the aws_db_instance

resource "aws_db_instance" "public_magnolia" {
  allocated_storage    = "${var.storage}"
  storage_type         = "${var.storage_type}"
  engine               = "${var.db_engine}"
  engine_version       = "${var.db_engine_version}"
  instance_class       = "${var.db_instance_type}"
  identifier           = "${var.customer_name}-${var.customer_environment}-public${count.index + 1}"
  db_subnet_group_name = "${var.db_subnet_group_name}"
  parameter_group_name = "${var.parameter_group_name}"
  username             = "${var.db_username}"
  password             = "${var.db_password}"
  name                 = "magnolia"
  count                = "${var.public_servers}"
  skip_final_snapshot  = true
  db_subnet_group_name = "${var.db_subnet_group_name}"
  availability_zone = "${(var.multi_az_deployment == "true")? var.az_list[count.index % length(var.az_list)] : ""}"
  multi_az = false
  vpc_security_group_ids = ["${aws_security_group.customer_security_group.id}"]
  publicly_accessible = false
  copy_tags_to_snapshot = true

  tags {
    Organization = "${var.organization}"
    Subscription = "${var.subscription}"
  }

  provisioner "local-exec" {
    command = <<EOT
      cd /var/provision/ansible &&
      export PATH=$PATH:/usr/share/python/facade/bin/ &&
      ansible-playbook -i "localhost," playbooks/create_magnolia_conf_sec_db.yml -e "db_address=${self.address} db_port=${self.port} db_username=${self.username} db_password=${self.password} rds_public_instance_name=${self.identifier}"
    EOT
  }
}

where az_list is passed from the caller module, as following

data "aws_availability_zones" "available" {
  state = "available"
}

module "main" {
  ...
  az_list = "${data.aws_availability_zones.available.names}"
  ...
}

While I agree that unset and "" would show up as a diff, I guess Terraform does not just base on the configuration diff, but also the actual calculated value, right? In term of value, unset and "" for AZ should be the same, I think.

[dohoangkhiem]

I think we made our case a little bit clearer, actually we already had the expression for availability_zone since before, and it's still the same

availability_zone = "${(var.multi_az_deployment == "true")? var.az_list[count.index % length(var.az_list)] : ""}"
with the actual value of multi_az_deployment is always "false"

As we checked with our old implementation, we used Terraform 0.9.8 and above expression didn't cause any changes in Terraform plan.

Then we upgraded to Terraform 0.10.7 to leverage the workspace, now with the same empty value of availability_zone Terraform always reports changes for RDS instance and wants to replace it.

[brandon-dacrib]

I ran into this problem today where tf kept wanting to recreate the rds cluster because the id value was generated. upgrading to 0.11.0 seems to have helped.
terraform --version
Terraform v0.11.0 + provider.aws v0.1.4

[foragerr]

@brandon-dacrib current aws provider version is 1.3.1, did you force the use of v0.1.4 to get around this issue? 0.1.4 seems.. ancient.

https://github.com/terraform-providers/terraform-provider-aws/releases

[brandon-dacrib]

No. That is what I had installed on my laptop. How do I force an upgrade?

[onwsk8r]

I've had this issue as well, both in 10.8, 11.0, and now 11.1 with AWS provider 1.2.0-1.5.0

The problem is that TF seems to get confused about the AZs the instances are using. Just now, it said it spun up databases in us-east-1b, c, and d, but it did not. They were in a, b, and c. I went to apply another change and it insisted on recreating the database instances. I specified a, b, and c as AZs as a workaround and it informed me there was no us-east-1a in this region- an error I've seen many times on many different configs. The DB subnet group it's using only exists in a, b and c. I changed the AZs to b, c, and d and it was all good. However, I just made another change to a different module, and now it's telling me that d doesn't exist. So I changed it to a. That also doesn't exist. Using just b and c is working, though :)

[Constantin07]

+1 facing the same issue - TF keeps to re-create rds instance.

$ terraform version
Terraform v0.11.1

$ terraform providers
.
├── provider.aws >= 1.5.0
├── provider.terraform

[baileydoestech]

+1 facing the same issue, also seems to be AZ related.

[MMarulla]

RDS Aurora seems to use all availability zones even if not requested. This causes the state file to be out of sync and forces a new resource. Example, only two availability zones requested:

resource "aws_rds_cluster" "myclustername" {
   ...
  availability_zones = [
    "${format("%sa", "${var.customer_region}")}",
    "${format("%sb", "${var.customer_region}")}"
  ]

Apply works as expected. After no changes, another apply generates this:

-/+ aws_rds_cluster.myclustername (new resource required)
  ...
      availability_zones.#:              "3" => "2" (forces new resource)
      availability_zones.1501760113:     "eu-west-2b" => "eu-west-2b"
      availability_zones.3230292939:     "eu-west-2a" => "eu-west-2a"
      availability_zones.780417767:      "eu-west-2c" => "" (forces new resource)

If the script is changed to:

resource "aws_rds_cluster" "myclustername" {
   ...
  availability_zones = [
    "${format("%sa", "${var.customer_region}")}",
    "${format("%sb", "${var.customer_region}")}",
    "${format("%sc", "${var.customer_region}")}"
  ]

Then no action is required on apply.

[finferflu]

Are there any updates on this? The comment by @MMarulla reflects exactly the issue I have. I also think RDS Aurora uses all availability zones by default, and it ignores whatever is set. In my configuration I have:

availability_zones     = ["eu-west-1b"]

And Terraform keeps outputting every single time:

availability_zones.#:                     "3" => "1" (forces new resource)
availability_zones.1924028850:            "eu-west-1b" => "eu-west-1b"
availability_zones.3953592328:            "eu-west-1a" => "" (forces new resource)
availability_zones.94988580:              "eu-west-1c" => "" (forces new resource)

Is there a temporary workaround to this, if not a fix? I only need to use one availability zone.

Thank you.

[MMarulla]

Here is what I ended up doing...

First, we set up as many subnets as there are AZ's available in the region, but no more than 4. In this example, the provider sets the region based on a variable from the tfvars file, and we set up (up to) four /27 subnets within a /24 CIDR block. The /24 block to use is also set in a variable in the tfvars file.

data "aws_availability_zones" "available" {
  provider = "aws.main"
}

locals {
  provider = "aws.main"
  numsubnets = "${length(data.aws_availability_zones.available.names) >= 4 ? 4 : length(data.aws_availability_zones.available.names)}"
  subnet_cidr_list = ["0", "32", "64", "96"]
}

resource "aws_subnet" "nonprod_subnets" {
 provider = "aws.main"
  count = "${local.numsubnets}"
  vpc_id  = "${aws_vpc.nonprod_vpc.id}"
  availability_zone = "${data.aws_availability_zones.available.names[count.index]}"
  cidr_block = "${format("10.81.%s.%s/27", var.cidr_24_block, local.subnet_cidr_list[count.index])}"
  tags {
    Name = "${format("%s NonProd Subnet %s", local.customer_name, upper(substr(data.aws_availability_zones.available.names[count.index], -1, 1)))}"
  }
}

Then, we set up the db_subnet to use however many subnets were created:

resource "aws_db_subnet_group" "nonprod" {
  provider = "aws.main"
  name       = "main"
  subnet_ids = ["${aws_subnet.nonprod_subnets.*.id}"]
}

Finally, in the aws_rds_cluster setup, we specify the db_subnet_group, but do not specify any availability zones at all. AWS spreads the Aurora assets across the subnets as it sees fit. This always works on re-apply and works in different regions with different numbers of availability zones.

[mwilbz]

We were observing changed "id" and "availability_zones" fields according to terraform plan output, when we were specifying 2 AZs and Terraform said the cluster was using 3. As @MMarulla suggested, we were able to fix the issue just by not passing an availability_zones argument in the aws_rds_cluster resource.

[ghost]

This issue has been automatically migrated to hashicorp/terraform-provider-aws#9760 because it looks like an issue with that provider. If you believe this is not an issue with the provider, please reply to hashicorp/terraform-provider-aws#9760.

[ghost]

I'm going to lock this issue because it has been closed for 30 days ⏳. This helps our maintainers find and focus on the active issues.

If you have found a problem that seems similar to this, please open a new issue and complete the issue template so we can capture all the details necessary to investigate further.