Ipv6 cluster dns ip

Makefile

@@ -29,35 +29,39 @@ T_YELLOW := \e[0;33m
 T_RESET := \e[0m
 
 .PHONY: all
-all: 1.19 1.20 1.21 1.22 1.23
+all: 1.19 1.20 1.21 1.22 1.23 ## Build all versions of EKS Optimized AL2 AMI
 
 .PHONY: validate
-validate:
+validate: ## Validate packer config
 	$(PACKER_BINARY) validate $(foreach packerVar,$(PACKER_VARIABLES), $(if $($(packerVar)),--var $(packerVar)='$($(packerVar))',)) eks-worker-al2.json
 
 .PHONY: k8s
-k8s: validate
+k8s: validate ## Build default K8s version of EKS Optimized AL2 AMI 
 	@echo "$(T_GREEN)Building AMI for version $(T_YELLOW)$(kubernetes_version)$(T_GREEN) on $(T_YELLOW)$(arch)$(T_RESET)"
 	$(PACKER_BINARY) build -timestamp-ui $(foreach packerVar,$(PACKER_VARIABLES), $(if $($(packerVar)),--var $(packerVar)='$($(packerVar))',)) eks-worker-al2.json
 
 # Build dates and versions taken from https://docs.aws.amazon.com/eks/latest/userguide/install-kubectl.html
 
 .PHONY: 1.19
-1.19:
+1.19: ## Build EKS Optimized AL2 AMI - K8s 1.19
 	$(MAKE) k8s kubernetes_version=1.19.15 kubernetes_build_date=2021-11-10 pull_cni_from_github=true
 
 .PHONY: 1.20
-1.20:
+1.20: ## Build EKS Optimized AL2 AMI - K8s 1.20
 	$(MAKE) k8s kubernetes_version=1.20.15 kubernetes_build_date=2022-07-27 pull_cni_from_github=true
 
 .PHONY: 1.21
-1.21:
+1.21: ## Build EKS Optimized AL2 AMI - K8s 1.21
 	$(MAKE) k8s kubernetes_version=1.21.14 kubernetes_build_date=2022-07-27 pull_cni_from_github=true
 
 .PHONY: 1.22
-1.22:
+1.22: ## Build EKS Optimized AL2 AMI - K8s 1.22
 	$(MAKE) k8s kubernetes_version=1.22.12 kubernetes_build_date=2022-07-27 pull_cni_from_github=true
 
 .PHONY: 1.23
-1.23:
+1.23: ## Build EKS Optimized AL2 AMI - K8s 1.23
 	$(MAKE) k8s kubernetes_version=1.23.9 kubernetes_build_date=2022-07-27 pull_cni_from_github=true
+
+.PHONY: help
+help: ## Display help
+	@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n  make \033[36m<target>\033[0m\n"} /^[\.a-zA-Z_0-9\-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

files/bootstrap.sh

@@ -151,6 +151,7 @@ IP_FAMILY="${IP_FAMILY:-}"
 SERVICE_IPV6_CIDR="${SERVICE_IPV6_CIDR:-}"
 ENABLE_LOCAL_OUTPOST="${ENABLE_LOCAL_OUTPOST:-}"
 CLUSTER_ID="${CLUSTER_ID:-}"
+IMDS_ENDPOINT="${IMDS_ENDPOINT:-169.254.169.254:80}"
 
 function get_pause_container_account_for_region () {
     local region="$1"
@@ -186,7 +187,7 @@ function _get_token() {
   local token_result=
   local http_result=
 
-  token_result=$(curl -s -w "\n%{http_code}" -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 600" "http://169.254.169.254/latest/api/token")
+  token_result=$(curl -s -w "\n%{http_code}" -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 600" "http://${IMDS_ENDPOINT}/latest/api/token")
   http_result=$(echo "$token_result" | tail -n 1)
   if [[ "$http_result" != "200" ]]
   then
@@ -218,11 +219,11 @@ function _get_meta_data() {
   local path=$1
   local metadata_result=
 
-  metadata_result=$(curl -s -w "\n%{http_code}" -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/$path)
+  metadata_result=$(curl -s -w "\n%{http_code}" -H "X-aws-ec2-metadata-token: $TOKEN" http://${IMDS_ENDPOINT}/$path)
   http_result=$(echo "$metadata_result" | tail -n 1)
   if [[ "$http_result" != "200" ]]
   then
-      echo -e "Failed to get metadata:\n$metadata_result\nhttp://169.254.169.254/$path\n$TOKEN"
+      echo -e "Failed to get metadata:\n$metadata_result\nhttp://${IMDS_ENDPOINT}/$path\n$TOKEN"
       return 1
   else
       local lines=$(echo "$metadata_result" | wc -l)
@@ -322,11 +323,6 @@ if [[ ! -z "${IP_FAMILY}" ]]; then
         echo "Invalid IpFamily. Only ipv4 or ipv6 are allowed"
         exit 1
   fi
-
-  if [[ "${IP_FAMILY}" == "ipv6" ]] && [[ ! -z "${B64_CLUSTER_CA}" ]] && [[ ! -z "${APISERVER_ENDPOINT}" ]] && [[ -z "${SERVICE_IPV6_CIDR}" ]]; then
-        echo "Service Ipv6 Cidr must be provided when ip-family is specified as IPV6"
-        exit 1
-  fi
 fi
 
 if [[ ! -z "${SERVICE_IPV6_CIDR}" ]]; then
@@ -339,7 +335,7 @@ fi
 
 TOKEN=$(get_token)
 AWS_DEFAULT_REGION=$(get_meta_data 'latest/dynamic/instance-identity/document' | jq .region -r)
-AWS_SERVICES_DOMAIN=$(get_meta_data '2018-09-24/meta-data/services/domain')
+AWS_SERVICES_DOMAIN=$(get_meta_data 'latest/meta-data/services/domain')
 
 MACHINE=$(uname -m)
 if [[ "$MACHINE" != "x86_64" && "$MACHINE" != "aarch64" ]]; then
@@ -410,7 +406,7 @@ if [[ -z "${B64_CLUSTER_CA}" ]] || [[ -z "${APISERVER_ENDPOINT}" ]]; then
 fi
 
 if [[ -z "${IP_FAMILY}" ]] || [[ "${IP_FAMILY}" == "None" ]]; then
-       ### this can happen when the ifFamily field is not found in describeCluster response
+       ### this can happen when the ipFamily field is not found in describeCluster response
        ### or B64_CLUSTER_CA and APISERVER_ENDPOINT are defined but IPFamily isn't
        IP_FAMILY="ipv4"
 fi
@@ -460,21 +456,28 @@ fi
 
 ### kubelet.service configuration
 
-if [[ "${IP_FAMILY}" == "ipv6" ]]; then
-      DNS_CLUSTER_IP=$(awk -F/ '{print $1}' <<< $SERVICE_IPV6_CIDR)a
-fi
-
 MAC=$(get_meta_data 'latest/meta-data/network/interfaces/macs/' | head -n 1 | sed 's/\/$//')
 
+
 if [[ -z "${DNS_CLUSTER_IP}" ]]; then
-  if [[ ! -z "${SERVICE_IPV4_CIDR}" ]] && [[ "${SERVICE_IPV4_CIDR}" != "None" ]] ; then
-    #Sets the DNS Cluster IP address that would be chosen from the serviceIpv4Cidr. (x.y.z.10)
-    DNS_CLUSTER_IP=${SERVICE_IPV4_CIDR%.*}.10
-  else
-    TEN_RANGE=$(get_meta_data "latest/meta-data/network/interfaces/macs/$MAC/vpc-ipv4-cidr-blocks" | grep -c '^10\..*' || true )
-    DNS_CLUSTER_IP=10.100.0.10
-    if [[ "$TEN_RANGE" != "0" ]]; then
-      DNS_CLUSTER_IP=172.20.0.10
+  if [[ "${IP_FAMILY}" == "ipv6" ]]; then
+    if [[ -z "${SERVICE_IPV6_CIDR}" ]]; then
+      echo "One of --service-ipv6-cidr or --dns-cluster-ip must be provided when ip-family is specified as ipv6"
+      exit 1
+    fi
+    DNS_CLUSTER_IP=$(awk -F/ '{print $1}' <<< $SERVICE_IPV6_CIDR)a
+  fi
+
+  if [[ "${IP_FAMILY}" == "ipv4" ]]; then
+    if [[ ! -z "${SERVICE_IPV4_CIDR}" ]] && [[ "${SERVICE_IPV4_CIDR}" != "None" ]]; then
+        #Sets the DNS Cluster IP address that would be chosen from the serviceIpv4Cidr. (x.y.z.10)
+        DNS_CLUSTER_IP=${SERVICE_IPV4_CIDR%.*}.10
+    else
+        TEN_RANGE=$(get_meta_data "latest/meta-data/network/interfaces/macs/$MAC/vpc-ipv4-cidr-blocks" | grep -c '^10\..*' || true )
+        DNS_CLUSTER_IP=10.100.0.10
+        if [[ "$TEN_RANGE" != "0" ]]; then
+            DNS_CLUSTER_IP=172.20.0.10
+        fi
     fi
   fi
 else

files/max-pods-calculator.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -o pipefail
 set -o nounset
@@ -76,14 +76,15 @@ CNI_MAX_ENI="${CNI_MAX_ENI:-}"
 INSTANCE_TYPE="${INSTANCE_TYPE:-}"
 INSTANCE_TYPE_FROM_IMDS="${INSTANCE_TYPE_FROM_IMDS:-false}"
 SHOW_MAX_ALLOWED="${SHOW_MAX_ALLOWED:-false}"
+IMDS_ENDPOINT="${IMDS_ENDPOINT:-169.254.169.254:80}"
 
 PREFIX_DELEGATION_SUPPORTED=false
 IPS_PER_PREFIX=16
 
 if [ "$INSTANCE_TYPE_FROM_IMDS" = true ]; then
-    TOKEN=$(curl -m 10 -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 600" -s "http://169.254.169.254/latest/api/token")
-    export AWS_DEFAULT_REGION=$(curl -s --retry 5 -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/dynamic/instance-identity/document | jq .region -r)
-    INSTANCE_TYPE=$(curl -m 10 -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-type)
+    TOKEN=$(curl -m 10 -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 600" -s "http://${IMDS_ENDPOINT}/latest/api/token")
+    export AWS_DEFAULT_REGION=$(curl -s --retry 5 -H "X-aws-ec2-metadata-token: $TOKEN" http://${IMDS_ENDPOINT}/latest/dynamic/instance-identity/document | jq .region -r)
+    INSTANCE_TYPE=$(curl -m 10 -H "X-aws-ec2-metadata-token: $TOKEN" -s http://${IMDS_ENDPOINT}/latest/meta-data/instance-type)
 elif [ -z "$INSTANCE_TYPE" ];
     # There's no reasonable default for an instanceType so force one to be provided to the script.
     then echo "You must specify an instance type to calculate max pods value."
@@ -119,7 +120,7 @@ if [[ "$CNI_MAJOR_VERSION" -gt 1 ]] || ([[ "$CNI_MAJOR_VERSION" = 1 ]] && [[ "$C
     PREFIX_DELEGATION_SUPPORTED=true
 fi
 
-DESCRIBE_INSTANCES_RESULT=$(aws ec2 describe-instance-types --instance-type $INSTANCE_TYPE --query 'InstanceTypes[0].{Hypervisor: Hypervisor, EniCount: NetworkInfo.MaximumNetworkInterfaces, PodsPerEniCount: NetworkInfo.Ipv4AddressesPerInterface, CpuCount: VCpuInfo.DefaultVCpus'} --output json)
+DESCRIBE_INSTANCES_RESULT=$(aws ec2 describe-instance-types --instance-type "${INSTANCE_TYPE}" --query 'InstanceTypes[0].{Hypervisor: Hypervisor, EniCount: NetworkInfo.MaximumNetworkInterfaces, PodsPerEniCount: NetworkInfo.Ipv4AddressesPerInterface, CpuCount: VCpuInfo.DefaultVCpus}' --output json)
 
 HYPERVISOR_TYPE=$(echo $DESCRIBE_INSTANCES_RESULT | jq -r '.Hypervisor' )
 IS_NITRO=false

test/Dockerfile

@@ -0,0 +1,15 @@
+FROM public.ecr.aws/aws-ec2/amazon-ec2-metadata-mock:v1.11.2 as aemm
+FROM public.ecr.aws/amazonlinux/amazonlinux:2
+
+ENV IMDS_ENDPOINT=127.0.0.1:1338
+COPY --from=aemm /ec2-metadata-mock /sbin/ec2-metadata-mock
+
+COPY files/kubelet-config.json /etc/kubernetes/kubelet/kubelet-config.json
+COPY files/kubelet-kubeconfig /var/lib/kubelet/kubeconfig
+COPY test/entrypoint.sh /entrypoint.sh
+COPY files /etc/eks
+COPY test/mocks/ /sbin/
+
+RUN yum install -y jq
+
+ENTRYPOINT ["/entrypoint.sh"]

test/README.md

@@ -0,0 +1,40 @@
+## Tests
+
+This directory contains a Dockerfile that is able to be used locally to test the `/etc/eks/boostrap.sh` script without having to use a real AL2 EC2 instance for a quick dev-loop. It is still necessary to test the bootstrap script on a real instance since the Docker image is not a fully accurate representation. 
+
+## AL2 EKS Optimized AMI Docker Image
+
+The image is built using the official AL2 image `public.ecr.aws/amazonlinux/amazonlinux:2`. It has several mocks installed including the [ec2-metadata-mock](https://github.com/aws/amazon-ec2-metadata-mock). Mocks are installed into `/sbin`, so adding addditional ones as necessary should be as simple as dropping a bash script in the `mocks` dir named as the command you would like to mock out. 
+
+## Usage
+
+```bash
+
+## The docker context needs to be at the root of the repo
+docker build -t eks-optimized-ami -f Dockerfile ../
+
+docker run -it eks-optimized-ami /etc/eks/bootstrap.sh --b64-cluster-ca dGVzdA== --apiserver-endpoint http://my-api-endpoint test
+```
+
+The `test-harness.sh` script wraps a build and runs test script in the `cases` dir. Tests scripts within the `cases` dir are invoked by the `test-harness.sh` script and have access to the `run` function. The `run` function accepts a temporary directory as an argument in order to mount as a volume in the container so that test scripts can check files within the `/etc/kubernetes/` directory after a bootstrap run. The remaining arguments to the `run` function are a path to a script within the AL2 EKS Optimized AMI Docker Container. 
+
+Here's an example `run` call:
+
+```
+run ${TEMP_DIR} /etc/eks/bootstrap.sh \
+    --b64-cluster-ca dGVzdA== \
+    --apiserver-endpoint http://my-api-endpoint \
+    --ip-family ipv4 \
+    --dns-cluster-ip 192.168.0.1 \
+    test-cluster-name
+```
+
+## ECR Public
+
+You may need to logout of ECR public or reauthenticate if your credentials are expired: 
+
+```bash
+docker logout public.ecr.aws
+```
+
+ECR public allow anonymous access, but you cannot have expired credentials loaded.

test/cases/ip-family-service-ipv6-cidr-mismatch.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "-> Should fail validation - ip-family mismatch"
+exit_code=0
+TEMP_DIR=$(mktemp -d)
+run ${TEMP_DIR} /etc/eks/bootstrap.sh \
+    --b64-cluster-ca dGVzdA== \
+    --apiserver-endpoint http://my-api-endpoint \
+    --ip-family ipv4 \
+    --service-ipv6-cidr 192.168.0.1/24 \
+    test || exit_code=$?
+
+if [[ ${exit_code} -eq 0 ]]; then
+    echo "❌ Test Failed: expected a non-zero exit code but got '${exit_code}'"
+    exit 1
+fi

test/cases/ipv4-cluster-dns-ip.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "--> Should return IPv4 DNS Cluster IP when given dns-cluster-ip"
+exit_code=0
+TEMP_DIR=$(mktemp -d)
+expected_cluster_dns="192.168.0.1"
+run ${TEMP_DIR} /etc/eks/bootstrap.sh \
+    --b64-cluster-ca dGVzdA== \
+    --apiserver-endpoint http://my-api-endpoint \
+    --ip-family ipv4 \
+    --dns-cluster-ip "${expected_cluster_dns}" \
+    test || exit_code=$?
+
+if [[ ${exit_code} -ne 0 ]]; then
+    echo "❌ Test Failed: expected a non-zero exit code but got '${exit_code}'"
+    exit 1
+fi
+
+actual_cluster_dns=$(jq -r '.clusterDNS[0]' < ${TEMP_DIR}/kubelet-config.json)
+if [[ ${actual_cluster_dns} != "${expected_cluster_dns}" ]]; then
+    echo "❌ Test Failed: expected clusterDNS IP '${expected_cluster_dns}' but got '${actual_cluster_dns}'"
+    exit 1
+fi

test/cases/ipv6-cluster-dns-ip.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "-> Should return ipv6 DNS Cluster IP when given dns-cluster-ip"
+exit_code=0
+TEMP_DIR=$(mktemp -d)
+expected_cluster_dns="fe80::2a"
+run ${TEMP_DIR} /etc/eks/bootstrap.sh \
+    --b64-cluster-ca dGVzdA== \
+    --apiserver-endpoint http://my-api-endpoint \
+    --ip-family ipv6 \
+    --dns-cluster-ip "${expected_cluster_dns}" \
+    test || exit_code=$?
+
+if [[ ${exit_code} -ne 0 ]]; then
+    echo "❌ Test Failed: expected a non-zero exit code but got '${exit_code}'"
+    exit 1
+fi
+
+actual_cluster_dns=$(jq -r '.clusterDNS[0]' < ${TEMP_DIR}/kubelet-config.json)
+if [[ ${actual_cluster_dns} != "${expected_cluster_dns}" ]]; then
+    echo "❌ Test Failed: expected clusterDNS IP '${expected_cluster_dns}' but got '${actual_cluster_dns}'"
+    exit 1
+fi

test/cases/ipv6-dns-cluster-ip-given-service-ipv6-cidr.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "-> Should return IPv6 DNS cluster IP when given service-ipv6-cidr"
+exit_code=0
+TEMP_DIR=$(mktemp -d)
+run ${TEMP_DIR} /etc/eks/bootstrap.sh \
+    --b64-cluster-ca dGVzdA== \
+    --apiserver-endpoint http://my-api-endpoint \
+    --ip-family ipv6 \
+    --service-ipv6-cidr fe80::1 \
+    test || exit_code=$?
+
+if [[ ${exit_code} -ne 0 ]]; then
+    echo "❌ Test Failed: expected a non-zero exit code but got '${exit_code}'"
+    exit 1
+fi
+
+expected_cluster_dns="fe80::1a"
+actual_cluster_dns=$(jq -r '.clusterDNS[0]' < ${TEMP_DIR}/kubelet-config.json)
+if [[ ${actual_cluster_dns} != "${expected_cluster_dns}" ]]; then
+    echo "❌ Test Failed: expected clusterDNS IP '${expected_cluster_dns}' but got '${actual_cluster_dns}'"
+    exit 1
+fi

test/cases/ipv6-ip-family-and-service-ipv6-cidr.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "-> Should fail w/ \"service-ipv6-cidr must be provided when ip-family is specified as ipv6\""
+exit_code=0
+TEMP_DIR=$(mktemp -d)
+run ${TEMP_DIR} /etc/eks/bootstrap.sh \
+    --b64-cluster-ca dGVzdA== \
+    --apiserver-endpoint http://my-api-endpoint \
+    --ip-family ipv6 \
+    test || exit_code=$?
+
+if [[ ${exit_code} -eq 0 ]]; then
+    echo "❌ Test Failed: expected a non-zero exit code but got '${exit_code}'"
+    exit 1
+fi

test/cases/max-pods-cni-1-11-2-delegation.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "-> Should calc max-pods successfully for VPC CNI 1.11.2"
+exit_code=0
+TEMP_DIR=$(mktemp -d)
+out=$(run ${TEMP_DIR} /etc/eks/max-pods-calculator.sh \
+    --instance-type m5.8xlarge \
+    --cni-version 1.11.2 \
+    --cni-prefix-delegation-enabled || exit_code=$?)
+echo $out
+
+if [[ ${exit_code} -ne 0 ]]; then
+    echo "❌ Test Failed: expected a non-zero exit code but got '${exit_code}'"
+    exit 1
+fi
+expected_max_pods="250"
+actual_max_pods=$(grep -o '[0-9]\+' <<< ${out})
+if [[ ${actual_max_pods} -ne ${expected_max_pods} ]]; then
+    echo "❌ Test Failed: expected max-pods for m4.xlarge w/ CNI 1.11.2 to be '${expected_max_pods}', but got '${actual_max_pods}'"
+    exit 1
+fi