Terraform destroy fails when Terraform apply failed

[varmax2511]

Terraform Version

Terraform v0.12.6

Terraform Configuration Files

variable "tenancy_ocid" {}
variable "user_ocid" {}
variable "fingerprint" {}
variable "private_key_path" {}
variable "region" {}

variable "compartment_ocid" {}
variable "ssh_public_key" {}


provider "oci" {
  tenancy_ocid     = "${var.tenancy_ocid}"
  user_ocid        = "${var.user_ocid}"
  fingerprint      = "${var.fingerprint}"
  private_key_path = "${var.private_key_path}"
  region           = "${var.region}"
}

# Defines the number of volumes to create and attach to each instance
# NOTE: Changing this value after applying it could result in re-attaching existing volumes to different instances.
# This is a result of using 'count' variables to specify the volume and instance IDs for the volume attachment resource.
variable "num_iscsi_volumes_per_instance" {
  default = "1"
}

variable "instance_shape" {
  default = "VM.Standard2.1"
}

variable "instance_image_ocid" {
  type = "map"

  default = {
    us-phoenix-1 = "ocid1.image.oc1.phx.aaaaaaaa4utqyolfcrdqef6u6aa3irshacsg55x7y6btsss6otqxzpvhshvq"
  }
}

variable "db_size" {
  default = "50" # size in GBs
}

variable "tag_namespace_description" {
  default = "Just a test"
}

variable "tag_namespace_name" {
  default = "testexamples-tag-namespace"
}

resource "oci_core_instance" "test_instance" {
  for_each             = var.instance_params
  availability_domain = "${data.oci_identity_availability_domain.ad.name}"
  compartment_id      = "${var.compartment_ocid}"
  display_name        = "jump"
  shape                = each.value.shape

  create_vnic_details {
    subnet_id        = "${oci_core_subnet.test_subnet.id}"
    display_name     = "Primaryvnic"
    assign_public_ip = true
    hostname_label   = each.value.hostname
  }

  source_details {
    source_type = "image"
    source_id   = "${var.instance_image_ocid[var.region]}"

    # Apply this to set the size of the boot volume that's created for this instance.
    # Otherwise, the default boot volume size of the image is used.
    # This should only be specified when source_type is set to "image".
    #boot_volume_size_in_gbs = "60"
  }

  # Apply the following flag only if you wish to preserve the attached boot volume upon destroying this instance
  # Setting this and destroying the instance will result in a boot volume that should be managed outside of this config.
  # When changing this value, make sure to run 'terraform apply' so that it takes effect before the resource is destroyed.
  #preserve_boot_volume = true

  metadata = {
    ssh_authorized_keys = "${var.ssh_public_key}"
    user_data           = "${base64encode(file("./userdata/bootstrap"))}"
  }

   timeouts {
    create = "60m"
  }
}

resource "oci_core_vcn" "test_vcn" {
  cidr_block     = "10.1.0.0/16"
  compartment_id = "${var.compartment_ocid}"
  display_name   = "TestVcn"
  dns_label      = "testvcn"
}

resource "oci_core_internet_gateway" "test_internet_gateway" {
  compartment_id = "${var.compartment_ocid}"
  display_name   = "TestInternetGateway"
  vcn_id         = "${oci_core_vcn.test_vcn.id}"
}

resource "oci_core_default_route_table" "default_route_table" {
  manage_default_resource_id = "${oci_core_vcn.test_vcn.default_route_table_id}"
  display_name               = "DefaultRouteTable"

  route_rules {
    destination       = "0.0.0.0/0"
    destination_type  = "CIDR_BLOCK"
    network_entity_id = "${oci_core_internet_gateway.test_internet_gateway.id}"
  }
}

resource "oci_core_subnet" "test_subnet" {
  availability_domain = "${data.oci_identity_availability_domain.ad.name}"
  cidr_block          = "10.1.20.0/24"
  display_name        = "TestSubnet"
  dns_label           = "testsubnet"
  security_list_ids   = ["${oci_core_vcn.test_vcn.default_security_list_id}"]
  compartment_id      = "${var.compartment_ocid}"
  vcn_id              = "${oci_core_vcn.test_vcn.id}"
  route_table_id      = "${oci_core_vcn.test_vcn.default_route_table_id}"
  dhcp_options_id     = "${oci_core_vcn.test_vcn.default_dhcp_options_id}"
}

data "oci_identity_availability_domain" "ad" {
  compartment_id = "${var.tenancy_ocid}"
  ad_number      = 1
}

resource "oci_core_volume" "block" {
  for_each            = var.bv_params
  availability_domain = oci_core_instance.test_instance[each.value.instance_name].availability_domain
  #compartment_id      = oci_core_instance.test_instance[each.value.instance_name].compartment_id
  display_name        = each.value.display_name
  size_in_gbs         = each.value.bv_size
  compartment_id      = oci_core_instance.test_instance[each.value.instance_name].compartment_id
}




variable "bv_params" {
  description = "Placeholder the bv parameters"
  type = map(object({
  display_name  = string
  bv_size       = number
  instance_name = string
  #device_name   = string
}))
}

variable "instance_params" {
description = "Placeholder for the parameters of the instances"
type = map(object({
shape                = string
hostname             = string
boot_volume_size     = number
assign_public_ip     = bool
private_ip           = string
nsg_names            = list(string)
preserve_boot_volume = bool
subnet_name          = string
device_disk_mappings = string
fault_domain         = number
image_name           = string
}))
}


terraform.tfvars
bv_params = {
  jumpbv1 = {
    display_name  = "jumpbv1"
    bv_size       = 50
    instance_name = "jump"
  }
}

instance_params = {
  jump = {
    shape                = "VM.Standard2.1"
    hostname             = "jump"
    boot_volume_size     = 300
    preserve_boot_volume = false
    assign_public_ip     = false
    private_ip           = "10.0.5.10"
    nsg_names            = ["nsg-test-jump"]
    subnet_name          = "test_jump"
    device_disk_mappings = "/u01:/dev/oracleoci/oraclevdb"
    fault_domain         = 1
    image_name           = "us-phoenix-1-win2016"
  }
}

Debug Output

Crash Output

Error: Invalid index
in resource "oci_core_volume" "block":
availability_domain = oci_core_instance. test_instance[each.value.instance_name].availability_domain
|----------------
| each.value.instance_name is "jump"
| oci_core_instance.test_instance is object with no attributes

The given key does not identify an element in this collection value.

Expected Behavior

Even though the apply failed, the destroy operation should have succeeded

Actual Behavior

When Apply failed at provisioning the instance (instance got in state Terminating from Provisioning, TF destroy also failed

Steps to Reproduce

Run Terraform apply
if apply fails
run TF destroy

Additional Context

References

[jbardin]

Hi @varmax2511,

There have been numerous fixes since the for_each feature was introduced in 0.12.6. Can you see if this is still reproducible with the latest release?

Thanks!

[varmax2511]

Hi @jbardin -- this is an intermittent issue and we are working on reproducing it. Meanwhile, below is the state file for the compute instance at the time of failure of destroy job

{

      "module": "module.instances-windows",

      "mode": "managed",

      "type": "oci_core_instance",

      "name": "this",

      "each": "map",

      "provider": "provider.oci.first",

      "instances": [

        {

          "index_key": "jump",

          "status": "tainted",

          "schema_version": 0,

          "attributes": {

            "agent_config": [

              {

                "is_management_disabled": false,

                "is_monitoring_disabled": false

              }

            ],

            "availability_domain": "......",

            "boot_volume_id": ".......",

            "compartment_id": ".......",

            "create_vnic_details": [

              {

                "assign_public_ip": "false",

                "defined_tags": {},

                "display_name": "",

                "freeform_tags": {},

                "hostname_label": "jump",

                "nsg_ids": [

                  "........"

                ],

                "private_ip": "........",

                "skip_source_dest_check": false,

                "subnet_id": "........."

              }

            ],

            "dedicated_vm_host_id": null,

            "defined_tags": {},

            "display_name": "jump",

            "extended_metadata": null,

            "fault_domain": ".....",

            "freeform_tags": {},

            "hostname_label": null,

            "id": "........",

            "image": ".........",

            "ipxe_script": null,

            "is_pv_encryption_in_transit_enabled": null,

            "launch_mode": "NATIVE",

            "launch_options": [

              {

                "boot_volume_type": "PARAVIRTUALIZED",

                "firmware": "UEFI_64",

                "is_consistent_volume_naming_enabled": false,

                "is_pv_encryption_in_transit_enabled": false,

                "network_type": "VFIO",

                "remote_data_volume_type": "PARAVIRTUALIZED"

              }

            ],

            "metadata": null,

            "preserve_boot_volume": false,

            "private_ip": null,

            "public_ip": null,

            "region": "....",

            "shape": "VM.Standard2.1",

            "source_details": [

              {

                "boot_volume_size_in_gbs": "300",

                "kms_key_id": "",

                "source_id": "........",

                "source_type": "image"

              }

            ],

            "state": "TERMINATING",

            "subnet_id": null,

            "system_tags": {},

            "time_created": "2020-01-16 18:17:51.892 +0000 UTC",

            "time_maintenance_reboot_due": "",

            "timeouts": null

          },

          "private": ".........."

        }

      ]

    },

State of the oci_core_volume and oci_core_volume_attachment

{

			"module": "module.instances-windows",

			"mode": "managed",

			"type": "oci_core_volume",

			"name": "block",

			"each": "map",

			"provider": "provider.oci.first",

			"instances": []

		},

		{

			"module": "module.instances-windows",

			"mode": "managed",

			"type": "oci_core_volume_attachment",

			"name": "attachment",

			"each": "map",

			"provider": "provider.oci.first",

			"instances": []

		}

[danieldreier]

This issue is also associated with an issue that was reported to support in ZenDesk, so the terraform core team is going to let support handle triage and reproduction and escalate it to engineering through the internal escalation process because support is already working with them on this.

[jbardin]

The failure here is during the refresh walk, which is partially driven by the configuration. Because data sources need to be fully re-evaluated during refresh, the entire config is loaded. Problems arise when the managed resource instances in the config do not match what is in the state. This leads to the situation where expressions from numerous places could require evaluating non-existent resource instances.

This class of issue is going to require refactoring the refresh cycle, probably consisting of the changes laid out in #17034.

Closing this one as a duplicate of #21096 to consolidate the discussion.

[ghost]

I'm going to lock this issue because it has been closed for 30 days ⏳. This helps our maintainers find and focus on the active issues.

If you have found a problem that seems similar to this, please open a new issue and complete the issue template so we can capture all the details necessary to investigate further.