Add ccm integration tests

cmd/kops/integration_test.go

@@ -350,6 +350,34 @@ func TestManyAddons(t *testing.T) {
 		runTestTerraformAWS(t)
 }
 
+func TestManyAddonsCCMIRSA(t *testing.T) {
+	featureflag.ParseFlags("+UseServiceAccountIAM,+EnableExternalCloudController")
+	unsetFeatureFlags := func() {
+		featureflag.ParseFlags("-UseServiceAccountIAM,-EnableExternalCloudController")
+	}
+	defer unsetFeatureFlags()
+
+	// We have to use a fixed CA because the fingerprint is inserted into the AWS WebIdentity configuration.
+	newIntegrationTest("minimal.example.com", "many-addons-ccm-irsa").
+		withCAKey().
+		withServiceAccountRole("dns-controller.kube-system", true).
+		withServiceAccountRole("aws-load-balancer-controller.kube-system", true).
+		withServiceAccountRole("cloud-controller-manager.kube-system", true).
+		withServiceAccountRole("cluster-autoscaler.kube-system", true).
+		withServiceAccountRole("ebs-csi-controller-sa.kube-system", true).
+		runTestTerraformAWS(t)
+}
+
+func TestCCM(t *testing.T) {
+	featureflag.ParseFlags("+EnableExternalCloudController")
+	unsetFeatureFlags := func() {
+		featureflag.ParseFlags("-EnableExternalCloudController")
+	}
+	defer unsetFeatureFlags()
+	newIntegrationTest("minimal.example.com", "many-addons-ccm").
+		runTestTerraformAWS(t)
+}
+
 // TestSharedSubnet runs the test on a configuration with a shared subnet (and VPC)
 func TestSharedSubnet(t *testing.T) {
 	newIntegrationTest("sharedsubnet.example.com", "shared_subnet").runTestTerraformAWS(t)

tests/integration/update_cluster/many-addons-ccm-irsa/README.md

@@ -0,0 +1,9 @@
+Simple test of (experimental) JWKS functionality
+
+We have to use a fixed CA because the fingerprint is inserted into the AWS WebIdentity configuration.
+
+ca.crt & ca.key generated with:
+
+```
+openssl req -new -newkey rsa:512 -days 3650 -nodes -x509 -subj "/CN=kubernetes" -keyout ca.key -out ca.crt -config <(cat /etc/ssl/openssl.cnf <(printf "[ v3_ca ]\nkeyUsage = critical,keyCertSign,cRLSign"))
+```

tests/integration/update_cluster/many-addons-ccm-irsa/ca.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBkTCCATugAwIBAgIUCpH+vP36aaPhoMAXYKNtGDRpO+0wDQYJKoZIhvcNAQEL
+BQAwFTETMBEGA1UEAwwKa3ViZXJuZXRlczAeFw0yMDA5MTIyMDE3MjhaFw0zMDA5
+MTAyMDE3MjhaMBUxEzARBgNVBAMMCmt1YmVybmV0ZXMwXDANBgkqhkiG9w0BAQEF
+AANLADBIAkEA4WWjrM1cq9lYsgmBYOZyjDaVYwCgb1zW4Bf5FMbWiWNuMjHPlVW2
+z17Q5ecKd0viUtF0A8/rrg3y7Lm0N3lIVwIDAQABo2MwYTAdBgNVHQ4EFgQU1d6Y
+G7ISO0T1baFPjv6ecnRFtJkwHwYDVR0jBBgwFoAU1d6YG7ISO0T1baFPjv6ecnRF
+tJkwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQEL
+BQADQQBG1IGyIUyg1/1JcqJv97CQdu2N+J/Ktgw7NIDsGwvYp4OW0y3mXSxWoIFk
+8l05a0McT3dLZawJ9VzpxMzJS4pG
+-----END CERTIFICATE-----

tests/integration/update_cluster/many-addons-ccm-irsa/ca.key

@@ -0,0 +1,10 @@
+-----BEGIN PRIVATE KEY-----
+MIIBVQIBADANBgkqhkiG9w0BAQEFAASCAT8wggE7AgEAAkEA4WWjrM1cq9lYsgmB
+YOZyjDaVYwCgb1zW4Bf5FMbWiWNuMjHPlVW2z17Q5ecKd0viUtF0A8/rrg3y7Lm0
+N3lIVwIDAQABAkAyOuFf6CAn1/bxLjcb7h9G6f8eogwe5TSpmg4TOEClOw0+Zy/y
+vgK2QlNQE0UPbpVXLVTr8/hKeExEpQpWhPoZAiEA91yvETWsBfhd14kiXXtROedu
+eeA7VFEKVAs3e6GkoeMCIQDpRJjgK1v66NRR0gWiDUknQg+O92BIX5SZ8F4CC4t5
+/QIhANUjwZ2cl6tVRNbxTPErzuOL7P+LHNQcOEAOojIfKBJtAiEAlJsN5WnaDCu9
+724kBov+OZNdRBAWd6Tkj3lQ+m6OaaUCIFiopekX5mvhslM7+ghbrwOTTY0Di1W9
++ZFYs9l9pitG
+-----END PRIVATE KEY-----

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_iam_role_aws-load-balancer-controller.kube-system.sa.minimal.example.com_policy

@@ -0,0 +1,17 @@
+{
+  "Statement": [
+    {
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "discovery.example.com/minimal.example.com:sub": "system:serviceaccount:kube-system:aws-load-balancer-controller"
+        }
+      },
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com"
+      }
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_iam_role_cloud-controller-manager.kube-system.sa.minimal.example.com_policy

@@ -0,0 +1,17 @@
+{
+  "Statement": [
+    {
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "discovery.example.com/minimal.example.com:sub": "system:serviceaccount:kube-system:cloud-controller-manager"
+        }
+      },
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com"
+      }
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_iam_role_cluster-autoscaler.kube-system.sa.minimal.example.com_policy

@@ -0,0 +1,17 @@
+{
+  "Statement": [
+    {
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "discovery.example.com/minimal.example.com:sub": "system:serviceaccount:kube-system:cluster-autoscaler"
+        }
+      },
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com"
+      }
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy

@@ -0,0 +1,17 @@
+{
+  "Statement": [
+    {
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "discovery.example.com/minimal.example.com:sub": "system:serviceaccount:kube-system:dns-controller"
+        }
+      },
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com"
+      }
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_iam_role_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy

@@ -0,0 +1,17 @@
+{
+  "Statement": [
+    {
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "discovery.example.com/minimal.example.com:sub": "system:serviceaccount:kube-system:ebs-csi-controller-sa"
+        }
+      },
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com"
+      }
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_iam_role_masters.minimal.example.com_policy

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_iam_role_nodes.minimal.example.com_policy

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_iam_role_policy_aws-load-balancer-controller.kube-system.sa.minimal.example.com_policy

@@ -0,0 +1,157 @@
+{
+  "Statement": [
+    {
+      "Action": "ec2:DescribeAvailabilityZones",
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:AuthorizeSecurityGroupIngress",
+        "ec2:DeleteSecurityGroup",
+        "ec2:RevokeSecurityGroupIngress",
+        "elasticloadbalancing:ModifyTargetGroupAttributes",
+        "elasticloadbalancing:ModifyRule",
+        "elasticloadbalancing:DeleteRule",
+        "elasticloadbalancing:AddTags",
+        "elasticloadbalancing:RemoveTags"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "aws:ResourceTag/elbv2.k8s.aws/cluster": "minimal.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "elasticloadbalancing:DescribeTags",
+        "elasticloadbalancing:DescribeTargetGroupAttributes",
+        "elasticloadbalancing:DescribeRules",
+        "elasticloadbalancing:DescribeTargetHealth",
+        "elasticloadbalancing:DescribeListenerCertificates",
+        "elasticloadbalancing:CreateRule"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:DescribeAccountAttributes",
+        "ec2:DescribeInstances",
+        "ec2:DescribeInternetGateways",
+        "ec2:DescribeRegions",
+        "ec2:DescribeRouteTables",
+        "ec2:DescribeSecurityGroups",
+        "ec2:DescribeSubnets",
+        "ec2:DescribeVolumes"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:CreateSecurityGroup",
+        "ec2:CreateTags",
+        "ec2:ModifyInstanceAttribute"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:AttachVolume",
+        "ec2:AuthorizeSecurityGroupIngress",
+        "ec2:CreateRoute",
+        "ec2:DeleteRoute",
+        "ec2:DeleteSecurityGroup",
+        "ec2:RevokeSecurityGroupIngress"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "ec2:ResourceTag/KubernetesCluster": "minimal.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:AttachVolume",
+        "ec2:AuthorizeSecurityGroupIngress",
+        "ec2:DeleteSecurityGroup",
+        "ec2:RevokeSecurityGroupIngress"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "ec2:ResourceTag/kubernetes.io/cluster/minimal.example.com": "owned"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "elasticloadbalancing:AddTags",
+        "elasticloadbalancing:AttachLoadBalancerToSubnets",
+        "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
+        "elasticloadbalancing:CreateLoadBalancer",
+        "elasticloadbalancing:CreateLoadBalancerPolicy",
+        "elasticloadbalancing:CreateLoadBalancerListeners",
+        "elasticloadbalancing:ConfigureHealthCheck",
+        "elasticloadbalancing:DeleteLoadBalancer",
+        "elasticloadbalancing:DeleteLoadBalancerListeners",
+        "elasticloadbalancing:DescribeLoadBalancers",
+        "elasticloadbalancing:DescribeLoadBalancerAttributes",
+        "elasticloadbalancing:DetachLoadBalancerFromSubnets",
+        "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
+        "elasticloadbalancing:ModifyLoadBalancerAttributes",
+        "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
+        "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:DescribeVpcs",
+        "elasticloadbalancing:AddTags",
+        "elasticloadbalancing:CreateListener",
+        "elasticloadbalancing:CreateTargetGroup",
+        "elasticloadbalancing:DeleteListener",
+        "elasticloadbalancing:DeleteTargetGroup",
+        "elasticloadbalancing:DeregisterTargets",
+        "elasticloadbalancing:DescribeListeners",
+        "elasticloadbalancing:DescribeLoadBalancerPolicies",
+        "elasticloadbalancing:DescribeTargetGroups",
+        "elasticloadbalancing:DescribeTargetHealth",
+        "elasticloadbalancing:ModifyListener",
+        "elasticloadbalancing:ModifyTargetGroup",
+        "elasticloadbalancing:RegisterTargets",
+        "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    }
+  ],
+  "Version": "2012-10-17"
+}