Merge pull request #19 from gruntwork-io/clean-up-images

Upgrade Packer and Cleanup Images

.circleci/config.yml

@@ -1,12 +1,12 @@
 defaults: &defaults
   machine: true
   environment:
-    GRUNTWORK_INSTALLER_VERSION: v0.0.21
+    GRUNTWORK_INSTALLER_VERSION: v0.0.23
     TERRATEST_LOG_PARSER_VERSION: v0.13.13
     MODULE_CI_VERSION: v0.13.3
     TERRAFORM_VERSION: 0.12.3
     TERRAGRUNT_VERSION: NONE
-    PACKER_VERSION: 1.3.2
+    PACKER_VERSION: 1.5.4
     GOLANG_VERSION: 1.11.2
 
 install_gruntwork_utils: &install_gruntwork_utils
@@ -45,7 +45,7 @@ jobs:
 
       # Fail the build if the pre-commit hooks don't pass. Note: if you run pre-commit install locally, these hooks will
       # execute automatically every time before you commit, ensuring the build never fails at this step!
-      - run: pip install pre-commit==1.11.2
+      - run: pip install pre-commit==1.11.2 cfgv==2.0.1
       - run: pre-commit install
       - run: pre-commit run --all-files
 
@@ -115,4 +115,3 @@ workflows:
       - test:
           requires:
             - build
-

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 repos:
   - repo: https://github.com/gruntwork-io/pre-commit
-    sha: v0.0.9
+    rev: v0.0.9
     hooks:
       - id: terraform-fmt

examples/machine-images/chronograf/chronograf.json

@@ -1,5 +1,5 @@
 {
-  "min_packer_version": "0.12.0",
+  "min_packer_version": "1.5.4",
   "variables": {
     "region": "us-east1",
     "zone": "us-east1-a",
@@ -16,7 +16,7 @@
     "region": "{{user `region`}}",
     "zone": "{{user `zone`}}",
     "image_description": "An Ubuntu 18.04 AMI that has Chronograf installed.",
-    "image_name": "chronograf-ubuntu-{{uuid | clean_image_name}}",
+    "image_name": "chronograf-ubuntu-{{uuid | clean_resource_name}}",
     "image_family": "chronograf-ubuntu-example",
     "account_file": "{{ user `service_account_json`}}",
     "ssh_username": "ubuntu"
@@ -29,15 +29,15 @@
       "sudo cp -r /tmp/bash-commons/modules/bash-commons/src /opt/gruntwork/bash-commons",
       "mkdir -p /tmp/terraform-google-influx/modules"
     ]
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/config",
     "destination": "/tmp"
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/../../../modules/",
     "destination": "/tmp/terraform-google-influx/modules"
-  },{
+  }, {
     "type": "shell",
     "inline": [
       "/tmp/terraform-google-influx/modules/install-chronograf/install-chronograf --version {{user `chronograf_version`}}"

examples/machine-images/influxdb-enterprise/influxdb-enterprise.json

@@ -1,5 +1,5 @@
 {
-  "min_packer_version": "0.12.0",
+  "min_packer_version": "1.5.4",
   "variables": {
     "region": "us-east1",
     "zone": "us-east1-a",
@@ -16,7 +16,7 @@
     "region": "{{user `region`}}",
     "zone": "{{user `zone`}}",
     "image_description": "An Ubuntu 18.04 AMI that has InfluxDB Enterprise installed.",
-    "image_name": "influxdb-ubuntu-ent-{{uuid | clean_image_name}}",
+    "image_name": "influxdb-ubuntu-ent-{{uuid | clean_resource_name}}",
     "image_family": "influxdb-ubuntu-ent-example",
     "account_file": "{{ user `service_account_json`}}",
     "ssh_username": "ubuntu"
@@ -28,23 +28,23 @@
       "sudo DEBIAN_FRONTEND=noninteractive apt-get update -y",
       "sudo apt-get install -y jq google-cloud-sdk"
     ]
-  },{
+  }, {
     "type": "shell",
     "inline": [
       "sudo mkdir -p /opt/gruntwork",
       "git clone --branch v0.1.2 https://github.com/gruntwork-io/bash-commons.git /tmp/bash-commons",
       "sudo cp -r /tmp/bash-commons/modules/bash-commons/src /opt/gruntwork/bash-commons",
       "mkdir -p /tmp/terraform-google-influx/modules"
     ]
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/config",
     "destination": "/tmp"
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/../../../modules/",
     "destination": "/tmp/terraform-google-influx/modules"
-  },{
+  }, {
     "type": "shell",
     "inline": [
       "/tmp/terraform-google-influx/modules/install-influxdb/install-influxdb --distribution enterprise --version {{user `version`}}"

examples/machine-images/influxdb-oss/influxdb-oss.json

@@ -1,5 +1,5 @@
 {
-  "min_packer_version": "0.12.0",
+  "min_packer_version": "1.5.4",
   "variables": {
     "region": "us-east1",
     "zone": "us-east1-a",
@@ -16,7 +16,7 @@
     "region": "{{user `region`}}",
     "zone": "{{user `zone`}}",
     "image_description": "An Ubuntu 18.04 AMI that has InfluxDB OSS installed.",
-    "image_name": "influxdb-oss-ubuntu-{{uuid | clean_image_name}}",
+    "image_name": "influxdb-oss-ubuntu-{{uuid | clean_resource_name}}",
     "image_family": "influxdb-ubuntu-oss-example",
     "account_file": "{{ user `service_account_json`}}",
     "ssh_username": "ubuntu"
@@ -28,23 +28,23 @@
       "sudo DEBIAN_FRONTEND=noninteractive apt-get update -y",
       "sudo apt-get install -y git"
     ]
-  },{
+  }, {
     "type": "shell",
     "inline": [
       "sudo mkdir -p /opt/gruntwork",
       "git clone --branch v0.1.2 https://github.com/gruntwork-io/bash-commons.git /tmp/bash-commons",
       "sudo cp -r /tmp/bash-commons/modules/bash-commons/src /opt/gruntwork/bash-commons",
       "mkdir -p /tmp/terraform-google-influx/modules"
     ]
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/config",
     "destination": "/tmp"
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/../../../modules/",
     "destination": "/tmp/terraform-google-influx/modules"
-  },{
+  }, {
     "type": "shell",
     "inline": [
       "/tmp/terraform-google-influx/modules/install-influxdb/install-influxdb --distribution oss --version {{user `version`}}"

examples/machine-images/kapacitor/kapacitor.json

@@ -1,5 +1,5 @@
 {
-  "min_packer_version": "0.12.0",
+  "min_packer_version": "1.5.4",
   "variables": {
     "region": "us-east1",
     "zone": "us-east1-a",
@@ -16,7 +16,7 @@
     "region": "{{user `region`}}",
     "zone": "{{user `zone`}}",
     "image_description": "An Ubuntu 18.04 AMI that has Kapacitor installed.",
-    "image_name": "kapacitor-ubuntu-{{uuid | clean_image_name}}",
+    "image_name": "kapacitor-ubuntu-{{uuid | clean_resource_name}}",
     "image_family": "kapacitor-ubuntu-example",
     "account_file": "{{ user `service_account_json`}}",
     "ssh_username": "ubuntu"
@@ -29,15 +29,15 @@
       "sudo cp -r /tmp/bash-commons/modules/bash-commons/src /opt/gruntwork/bash-commons",
       "mkdir -p /tmp/terraform-google-influx/modules"
     ]
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/config",
     "destination": "/tmp"
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/../../../modules/",
     "destination": "/tmp/terraform-google-influx/modules"
-  },{
+  }, {
     "type": "shell",
     "inline": [
       "/tmp/terraform-google-influx/modules/install-kapacitor/install-kapacitor --version {{user `kapacitor_version`}}"

examples/machine-images/telegraf/telegraf.json

@@ -1,5 +1,5 @@
 {
-  "min_packer_version": "0.12.0",
+  "min_packer_version": "1.5.4",
   "variables": {
     "region": "us-east1",
     "zone": "us-east1-a",
@@ -16,7 +16,7 @@
     "region": "{{user `region`}}",
     "zone": "{{user `zone`}}",
     "image_description": "An Ubuntu 18.04 AMI that has Telegraf installed.",
-    "image_name": "telegraf-ubuntu-{{uuid | clean_image_name}}",
+    "image_name": "telegraf-ubuntu-{{uuid | clean_resource_name}}",
     "image_family": "telegraf-ubuntu-example",
     "account_file": "{{ user `service_account_json`}}",
     "ssh_username": "ubuntu"
@@ -29,15 +29,15 @@
       "sudo cp -r /tmp/bash-commons/modules/bash-commons/src /opt/gruntwork/bash-commons",
       "mkdir -p /tmp/terraform-google-influx/modules"
     ]
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/config",
     "destination": "/tmp"
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/../../../modules/",
     "destination": "/tmp/terraform-google-influx/modules"
-  },{
+  }, {
     "type": "shell",
     "inline": [
       "/tmp/terraform-google-influx/modules/install-telegraf/install-telegraf --version {{user `version`}}"

examples/machine-images/tick-oss-all-in-one/tick-oss.json

@@ -1,5 +1,5 @@
 {
-  "min_packer_version": "0.12.0",
+  "min_packer_version": "1.5.4",
   "variables": {
     "region": "us-east1",
     "zone": "us-east1-b",
@@ -19,7 +19,7 @@
     "region": "{{user `region`}}",
     "zone": "{{user `zone`}}",
     "image_description": "An Ubuntu 18.04 AMI that has TICK Stack components (OSS) installed.",
-    "image_name": "tick-oss-ubuntu-{{uuid | clean_image_name}}",
+    "image_name": "tick-oss-ubuntu-{{uuid | clean_resource_name}}",
     "image_family": "tick-ubuntu-oss-example",
     "account_file": "{{ user `service_account_json`}}",
     "ssh_username": "ubuntu"
@@ -31,35 +31,35 @@
       "sudo DEBIAN_FRONTEND=noninteractive apt-get update -y",
       "sudo apt-get install -y git"
     ]
-  },{
+  }, {
     "type": "shell",
     "inline": [
       "sudo mkdir -p /opt/gruntwork",
       "git clone --branch v0.1.2 https://github.com/gruntwork-io/bash-commons.git /tmp/bash-commons",
       "sudo cp -r /tmp/bash-commons/modules/bash-commons/src /opt/gruntwork/bash-commons",
       "mkdir -p /tmp/terraform-google-influx/modules"
     ]
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/../../../modules/",
     "destination": "/tmp/terraform-google-influx/modules"
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/../influxdb-oss/config",
     "destination": "/tmp"
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/../kapacitor/config",
     "destination": "/tmp"
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/../chronograf/config",
     "destination": "/tmp"
-  },{
+  }, {
     "type": "file",
     "source": "{{template_dir}}/../telegraf/config",
     "destination": "/tmp"
-  },{
+  }, {
     "type": "shell",
     "inline": [
       "/tmp/terraform-google-influx/modules/install-influxdb/install-influxdb --distribution oss --version {{user `influxdb_version`}}",

test/influxdb_enterprise_test.go

@@ -2,17 +2,18 @@ package test
 
 import (
 	"fmt"
-	"github.com/stretchr/testify/require"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/gruntwork-io/terratest/modules/gcp"
 	"github.com/gruntwork-io/terratest/modules/random"
 	"github.com/gruntwork-io/terratest/modules/terraform"
-	"github.com/gruntwork-io/terratest/modules/test-structure"
+	test_structure "github.com/gruntwork-io/terratest/modules/test-structure"
 )
 
 const EXAMPLE_DIR_INFLUXDB_ENTERPRISE = "influxdb-enterprise"
@@ -74,12 +75,15 @@ func TestInfluxDBEnterprise(t *testing.T) {
 				test_structure.SaveString(t, exampleDir, KEY_ZONE, zone)
 				test_structure.SaveString(t, exampleDir, KEY_PROJECT, projectId)
 				test_structure.SaveString(t, exampleDir, KEY_RANDOM_ID, randomId)
-
 			})
 
 			defer test_structure.RunTestStage(t, "teardown", func() {
 				terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir)
+				projectId := test_structure.LoadString(t, exampleDir, KEY_PROJECT)
 				terraform.Destroy(t, terraformOptions)
+
+				imageName := test_structure.LoadArtifactID(t, exampleDir)
+				deleteImage(t, projectId, imageName)
 			})
 
 			test_structure.RunTestStage(t, "build_image", func() {
@@ -98,6 +102,7 @@ func TestInfluxDBEnterprise(t *testing.T) {
 				templatePath := fmt.Sprintf("%s/%s", imagesDir, testCase.packerInfo.templatePath)
 
 				imageID := buildImage(t, templatePath, testCase.packerInfo.builderName, projectId, region, zone)
+				test_structure.SaveArtifactID(t, exampleDir, imageID)
 
 				baseName := "influxdb-ent"
 

test/influxdb_oss_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/gruntwork-io/terratest/modules/gcp"
 	"github.com/gruntwork-io/terratest/modules/random"
 	"github.com/gruntwork-io/terratest/modules/terraform"
-	"github.com/gruntwork-io/terratest/modules/test-structure"
+	test_structure "github.com/gruntwork-io/terratest/modules/test-structure"
 )
 
 const EXAMPLE_DIR_INFLUXDB_OSS = "influxdb-oss"
@@ -72,12 +72,15 @@ func TestInfluxDBOSS(t *testing.T) {
 				test_structure.SaveString(t, exampleDir, KEY_ZONE, zone)
 				test_structure.SaveString(t, exampleDir, KEY_PROJECT, projectId)
 				test_structure.SaveString(t, exampleDir, KEY_RANDOM_ID, randomId)
-
 			})
 
 			defer test_structure.RunTestStage(t, "teardown", func() {
 				terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir)
+				projectId := test_structure.LoadString(t, exampleDir, KEY_PROJECT)
 				terraform.Destroy(t, terraformOptions)
+
+				imageName := test_structure.LoadArtifactID(t, exampleDir)
+				deleteImage(t, projectId, imageName)
 			})
 
 			test_structure.RunTestStage(t, "build_image", func() {
@@ -90,6 +93,7 @@ func TestInfluxDBOSS(t *testing.T) {
 				templatePath := fmt.Sprintf("%s/%s", imagesDir, testCase.packerInfo.templatePath)
 
 				imageID := buildImage(t, templatePath, testCase.packerInfo.builderName, projectId, region, zone)
+				test_structure.SaveArtifactID(t, exampleDir, imageID)
 
 				baseName := "influxdb-oss"
 

test/test_helpers.go

@@ -4,13 +4,14 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/gruntwork-io/terratest/modules/gcp"
-	"github.com/gruntwork-io/terratest/modules/terraform"
-	test_structure "github.com/gruntwork-io/terratest/modules/test-structure"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/gruntwork-io/terratest/modules/gcp"
+	"github.com/gruntwork-io/terratest/modules/terraform"
+	test_structure "github.com/gruntwork-io/terratest/modules/test-structure"
+
 	"github.com/gruntwork-io/terratest/modules/logger"
 	"github.com/gruntwork-io/terratest/modules/retry"
 
@@ -62,6 +63,11 @@ func buildImage(t *testing.T, templatePath string, builderName string, project s
 	return packer.BuildArtifact(t, options)
 }
 
+func deleteImage(t *testing.T, project string, imageName string) {
+	image := gcp.FetchImage(t, project, imageName)
+	image.DeleteImage(t)
+}
+
 func getNodePublicIP(t *testing.T, exampleDir string, outputName string) string {
 	maxRetries := 15
 	sleepBetweenRetries := 5 * time.Second