Automated cherry pick of #10130: Add verify-cloudformation script #10131: Fix cloudformation lint errors

Makefile

@@ -458,6 +458,10 @@ verify-shellcheck:
 verify-terraform:
 	./hack/verify-terraform.sh
 
+.PHONE: verify-cloudformation
+verify-cloudformation:
+	hack/verify-cloudformation.sh
+
 .PHONY: verify-bindata
 verify-bindata:
 	./hack/verify-bindata.sh

hack/.cfnlintrc.yaml

@@ -0,0 +1,3 @@
+ignore_checks:
+- W3010 # Don't hardcode AZs
+- E2510 # us-test-1a is not a valid AZ

hack/verify-cloudformation.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+. "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+TAG=v0.39.0
+IMAGE="cfn-python-lint:${TAG}"
+
+# There is no official docker image so build it locally
+# https://github.com/aws-cloudformation/cfn-python-lint/issues/1025
+function docker_build() {
+  echo "Building cfn-python-lint image"
+  TMP=$(mktemp -d)
+  git clone -q -b "${TAG}" https://github.com/aws-cloudformation/cfn-python-lint "${TMP}"
+  pushd "${TMP}"
+  docker build --tag "${IMAGE}" .
+  popd
+  rm -rf "${TMP}"
+}
+
+docker image inspect "${IMAGE}" >/dev/null 2>&1 || docker_build
+
+docker run --rm -v "${KOPS_ROOT}:/${KOPS_ROOT}" -v "${KOPS_ROOT}/hack/.cfnlintrc.yaml:/root/.cfnlintrc" "${IMAGE}" "/${KOPS_ROOT}/tests/integration/update_cluster/**/cloudformation.json"
+RC=$?
+
+if [ $RC != 0 ]; then
+  echo -e "\nCloudformation linting failed\n"
+  exit 0 # TODO: exit $RC once issues have been addressed to make this a blocking check
+else
+  echo -e "\nCloudformation linting succeeded\n"
+fi

tests/integration/update_cluster/complex/cloudformation.json

@@ -640,7 +640,7 @@
         "FromPort": 443,
         "ToPort": 443,
         "IpProtocol": "tcp",
-        "CidrIp": "2001:0:8500::/40"
+        "CidrIpv6": "2001:0:8500::/40"
       }
     },
     "AWSEC2SecurityGroupIngresshttpselbtomaster": {
@@ -678,7 +678,7 @@
         "FromPort": 3,
         "ToPort": 4,
         "IpProtocol": "icmp",
-        "CidrIp": "2001:0:8500::/40"
+        "CidrIpv6": "2001:0:8500::/40"
       }
     },
     "AWSEC2SecurityGroupIngressnodeporttcpexternaltonode102030024": {
@@ -806,7 +806,7 @@
         "FromPort": 22,
         "ToPort": 22,
         "IpProtocol": "tcp",
-        "CidrIp": "2001:0:85a3::/48"
+        "CidrIpv6": "2001:0:85a3::/48"
       }
     },
     "AWSEC2SecurityGroupIngresssshexternaltonode111132": {
@@ -830,7 +830,7 @@
         "FromPort": 22,
         "ToPort": 22,
         "IpProtocol": "tcp",
-        "CidrIp": "2001:0:85a3::/48"
+        "CidrIpv6": "2001:0:85a3::/48"
       }
     },
     "AWSEC2SecurityGroupapielbcomplexexamplecom": {

tests/integration/update_cluster/mixed_instances/cloudformation.json

@@ -318,7 +318,7 @@
           },
           "InstancesDistribution": {
             "OnDemandPercentageAboveBaseCapacity": 5,
-            "SpotInstancePool": 3
+            "SpotInstancePools": 3
           }
         }
       }

tests/integration/update_cluster/mixed_instances_spot/cloudformation.json

@@ -318,7 +318,7 @@
           },
           "InstancesDistribution": {
             "OnDemandPercentageAboveBaseCapacity": 5,
-            "SpotInstancePool": 3,
+            "SpotInstancePools": 3,
             "SpotMaxPrice": "0.1"
           }
         }

upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go

@@ -846,7 +846,7 @@ type cloudformationAutoscalingInstanceDistribution struct {
 	// SpotAllocationStrategy is the spot allocation stratergy
 	SpotAllocationStrategy *string `json:"SpotAllocationStrategy,omitempty"`
 	// SpotInstancePool is the number of pools
-	SpotInstancePool *int64 `json:"SpotInstancePool,omitempty"`
+	SpotInstancePools *int64 `json:"SpotInstancePools,omitempty"`
 	// SpotMaxPrice is the max bid on spot instance, defaults to demand value
 	SpotMaxPrice *string `json:"SpotMaxPrice,omitempty"`
 }
@@ -901,7 +901,7 @@ func (_ *AutoscalingGroup) RenderCloudformation(t *cloudformation.Cloudformation
 				OnDemandBaseCapacity:                e.MixedOnDemandBase,
 				OnDemandPercentageAboveBaseCapacity: e.MixedOnDemandAboveBase,
 				SpotAllocationStrategy:              e.MixedSpotAllocationStrategy,
-				SpotInstancePool:                    e.MixedSpotInstancePools,
+				SpotInstancePools:                   e.MixedSpotInstancePools,
 				SpotMaxPrice:                        e.MixedSpotMaxPrice,
 			},
 		}

upup/pkg/fi/cloudup/awstasks/securitygrouprule.go

@@ -348,6 +348,7 @@ type cloudformationSecurityGroupIngress struct {
 
 	Protocol *string `json:"IpProtocol,omitempty"`
 	CidrIp   *string `json:"CidrIp,omitempty"`
+	CidrIpv6 *string `json:"CidrIpv6,omitempty"`
 }
 
 func (_ *SecurityGroupRule) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *SecurityGroupRule) error {
@@ -383,7 +384,11 @@ func (_ *SecurityGroupRule) RenderCloudformation(t *cloudformation.Cloudformatio
 	}
 
 	if e.CIDR != nil {
-		tf.CidrIp = e.CIDR
+		if strings.Contains(fi.StringValue(e.CIDR), ":") {
+			tf.CidrIpv6 = e.CIDR
+		} else {
+			tf.CidrIp = e.CIDR
+		}
 	}
 
 	return t.RenderResource(cfType, *e.Name, tf)