add terraform testing

kubernetes · Mar 31, 2020 · a0e1672 · a0e1672
1 parent 8f8d7af
commit a0e1672
Show file tree

Hide file tree

Showing 248 changed files with 22,967 additions and 43 deletions.
diff --git a/Makefile b/Makefile
@@ -154,6 +154,7 @@ help: # Show this help
 clean: # Remove build directory and bindata-generated files
 	for t in ${BINDATA_TARGETS}; do if test -e $$t; then rm -fv $$t; fi; done
 	if test -e ${BUILD}; then rm -rfv ${BUILD}; fi
+	rm -rf tests/integration/update_cluster/*/.terraform
 
 .PHONY: kops
 kops: ${KOPS}
@@ -210,11 +211,6 @@ hooks: # Install Git hooks
 test: ${BINDATA_TARGETS}  # Run tests locally
 	go test -v ./...
 
-.PHONY: terraform-validate
-terraform-validate:
-	# TODO: loop over all update_cluster directories
-	docker run --rm -it -v ${KOPS_ROOT}/tests/integration/update_cluster/complex:/tf -w /tf --entrypoint=sh hashicorp/terraform:0.11.14 -c '/bin/terraform init && /bin/terraform validate'
-
 .PHONY: ${DIST}/linux/amd64/nodeup
 ${DIST}/linux/amd64/nodeup: ${BINDATA_TARGETS}
 	mkdir -p ${DIST}
@@ -543,12 +539,16 @@ verify-staticcheck: ${BINDATA_TARGETS}
 verify-shellcheck:
 	${KOPS_ROOT}/hack/verify-shellcheck.sh
 
+.PHONY: verify-terraform
+verify-terraform:
+	./hack/verify-terraform.sh
+
 # ci target is for developers, it aims to cover all the CI jobs
 # verify-gendocs will call kops target
 # verify-package has to be after verify-gendocs, because with .gitignore for federation bindata
 # it bombs in travis. verify-gendocs generates the bindata file.
 .PHONY: ci
-ci: govet verify-gofmt verify-generate verify-gomod verify-goimports verify-boilerplate verify-bazel verify-misspelling verify-shellcheck verify-staticcheck nodeup examples test | verify-gendocs verify-packages verify-apimachinery
+ci: govet verify-gofmt verify-generate verify-gomod verify-goimports verify-boilerplate verify-bazel verify-misspelling verify-shellcheck verify-staticcheck verify-terraform nodeup examples test | verify-gendocs verify-packages verify-apimachinery
 	echo "Done!"
 
 # travis-ci is the target that travis-ci calls

diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go
@@ -448,35 +448,14 @@ func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarn
 		// are actually produced), validate that the provided expected data file
 		// contents match actual data file content
 		expectedDataPath := path.Join(i.srcDir, "data")
-		if _, err := os.Stat(expectedDataPath); err == nil {
-			expectedDataFiles, err := ioutil.ReadDir(expectedDataPath)
-			if err != nil {
-				t.Fatalf("failed to read expected data dir: %v", err)
-			}
-			for _, expectedDataFile := range expectedDataFiles {
-				dataFileName := expectedDataFile.Name()
-				expectedDataContent, err :=
-					ioutil.ReadFile(path.Join(expectedDataPath, dataFileName))
-				if err != nil {
-					t.Fatalf("failed to read expected data file: %v", err)
-				}
+		{
+			for _, dataFileName := range expectedDataFilenames {
 				actualDataContent, err :=
 					ioutil.ReadFile(path.Join(actualDataPath, dataFileName))
 				if err != nil {
 					t.Fatalf("failed to read actual data file: %v", err)
 				}
-				if string(expectedDataContent) != string(actualDataContent) {
-					t.Fatalf(
-						"actual data file (%s) did not match the content of expected data file (%s). "+
-							"NOTE: If outputs seem identical, check for end-of-line differences, "+
-							"especially if the file is in multipart MIME format!"+
-							"\nBEGIN_ACTUAL:\n%s\nEND_ACTUAL\nBEGIN_EXPECTED:\n%s\nEND_EXPECTED",
-						path.Join(actualDataPath, dataFileName),
-						path.Join(expectedDataPath, dataFileName),
-						actualDataContent,
-						expectedDataContent,
-					)
-				}
+				golden.AssertMatchesFile(t, string(actualDataContent), path.Join(expectedDataPath, dataFileName))
 			}
 		}
 	}

diff --git a/hack/verify-terraform.sh b/hack/verify-terraform.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+. "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+# integration test cluster directories that are terraform 0.12 compatible
+CLUSTERS_0_12=(
+  "minimal-json"
+)
+
+# Terraform versions
+TAG_0_12=0.12.23
+TAG_0_11=0.11.14
+
+RC=0
+while IFS= read -r -d '' -u 3 test_dir; do
+  [ -f "${test_dir}/kubernetes.tf" ] || [ -f "${test_dir}/kubernetes.tf.json" ] || continue
+  echo -e "${test_dir}\n"
+  cluster=$(basename "${test_dir}")
+  kube::util::array_contains "${cluster}" "${CLUSTERS_0_12[@]}" && tag=$TAG_0_12 || tag=$TAG_0_11
+
+  docker run --rm -it -v "${test_dir}":"${test_dir}" -w "${test_dir}" --entrypoint=sh hashicorp/terraform:$tag -c '/bin/terraform init >/dev/null && /bin/terraform validate' || RC=$?
+done 3< <(find "${KOPS_ROOT}/tests/integration/update_cluster" -type d -maxdepth 1 -print0)
+
+if [ $RC != 0 ]; then
+  echo -e "\nTerraform validation failed\n"
+  # TODO(rifelpet): make this script blocking in PRs by exiting non-zero on failure
+  # exit $RC
+  exit 0
+else
+  echo -e "\nTerraform validation succeeded\n"
+fi
diff --git a/pkg/testutils/golden/compare.go b/pkg/testutils/golden/compare.go
@@ -19,6 +19,7 @@ package golden
 import (
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -42,8 +43,9 @@ func AssertMatchesFile(t *testing.T, actual string, p string) {
 
 	//on windows, with git set to autocrlf, the reference files on disk have windows line endings
 	expected = strings.Replace(expected, "\r\n", "\n", -1)
+	actual = strings.Replace(actual, "\r\n", "\n", -1)
 
-	if actual == expected {
+	if actual == expected && err == nil {
 		return
 	}
 
@@ -52,7 +54,9 @@ func AssertMatchesFile(t *testing.T, actual string, p string) {
 
 		// Keep git happy with a trailing newline
 		actual += "\n"
-
+		if err := os.MkdirAll(path.Dir(p), 0755); err != nil {
+			t.Errorf("error creating directory %s: %v", path.Dir(p), err)
+		}
 		if err := ioutil.WriteFile(p, []byte(actual), 0644); err != nil {
 			t.Errorf("error writing expected output %s: %v", p, err)
 		}

diff --git a/...pdate_cluster/additional_cidr/data/aws_iam_role_masters.additionalcidr.example.com_policy b/...pdate_cluster/additional_cidr/data/aws_iam_role_masters.additionalcidr.example.com_policy
@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
diff --git a/.../update_cluster/additional_cidr/data/aws_iam_role_nodes.additionalcidr.example.com_policy b/.../update_cluster/additional_cidr/data/aws_iam_role_nodes.additionalcidr.example.com_policy
@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
diff --git a/...luster/additional_cidr/data/aws_iam_role_policy_masters.additionalcidr.example.com_policy b/...luster/additional_cidr/data/aws_iam_role_policy_masters.additionalcidr.example.com_policy
@@ -0,0 +1,102 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:*"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "autoscaling:DescribeAutoScalingGroups",
+        "autoscaling:DescribeAutoScalingInstances",
+        "autoscaling:DescribeLaunchConfigurations",
+        "autoscaling:DescribeTags",
+        "autoscaling:SetDesiredCapacity",
+        "autoscaling:TerminateInstanceInAutoScalingGroup",
+        "autoscaling:UpdateAutoScalingGroup",
+        "ec2:DescribeLaunchTemplateVersions"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "elasticloadbalancing:*"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "iam:ListServerCertificates",
+        "iam:GetServerCertificate"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "route53:ChangeResourceRecordSets",
+        "route53:ListResourceRecordSets",
+        "route53:GetHostedZone"
+      ],
+      "Resource": [
+        "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "route53:GetChange"
+      ],
+      "Resource": [
+        "arn:aws:route53:::change/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "route53:ListHostedZones"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "route53:ListHostedZones"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecr:GetAuthorizationToken",
+        "ecr:BatchCheckLayerAvailability",
+        "ecr:GetDownloadUrlForLayer",
+        "ecr:GetRepositoryPolicy",
+        "ecr:DescribeRepositories",
+        "ecr:ListImages",
+        "ecr:BatchGetImage"
+      ],
+      "Resource": [
+        "*"
+      ]
+    }
+  ]
+}
diff --git a/..._cluster/additional_cidr/data/aws_iam_role_policy_nodes.additionalcidr.example.com_policy b/..._cluster/additional_cidr/data/aws_iam_role_policy_nodes.additionalcidr.example.com_policy
@@ -0,0 +1,68 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:DescribeInstances",
+        "ec2:DescribeRegions"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "route53:ChangeResourceRecordSets",
+        "route53:ListResourceRecordSets",
+        "route53:GetHostedZone"
+      ],
+      "Resource": [
+        "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "route53:GetChange"
+      ],
+      "Resource": [
+        "arn:aws:route53:::change/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "route53:ListHostedZones"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "route53:ListHostedZones"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecr:GetAuthorizationToken",
+        "ecr:BatchCheckLayerAvailability",
+        "ecr:GetDownloadUrlForLayer",
+        "ecr:GetRepositoryPolicy",
+        "ecr:DescribeRepositories",
+        "ecr:ListImages",
+        "ecr:BatchGetImage"
+      ],
+      "Resource": [
+        "*"
+      ]
+    }
+  ]
+}
diff --git a/...ey_pair_kubernetes.additionalcidr.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key b/...ey_pair_kubernetes.additionalcidr.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==