From f446b4b53b57cd08cad3a125cc0518faf31871ef Mon Sep 17 00:00:00 2001 From: Justin Garrison Date: Tue, 29 Jun 2021 21:02:28 -0700 Subject: [PATCH] Add getting started based on fargate (#488) * Add getting started based on fargate * Removed "Notes" and clarified fargate * Remove LATEST_ from karpenter version * Minor edits and fp profile update --- docs/aws/README.md | 99 +++++++++++++++----------- docs/aws/eks-config.yaml | 43 +++++++++++ docs/aws/karpenter.cloudformation.yaml | 54 -------------- 3 files changed, 99 insertions(+), 97 deletions(-) create mode 100644 docs/aws/eks-config.yaml diff --git a/docs/aws/README.md b/docs/aws/README.md index 1ebfb6031c76..a9be1e501a33 100644 --- a/docs/aws/README.md +++ b/docs/aws/README.md @@ -3,31 +3,32 @@ This guide will provide a complete Karpenter installation for AWS. These steps are opinionated and may need to be adapted for your use case. +> This guide should take less than 1 hour to complete and cost less than $.25 + ## Environment ```bash -CLOUD_PROVIDER=aws +export CLUSTER_NAME=$USER-karpenter-demo +export AWS_DEFAULT_REGION=us-west-2 AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) -CLUSTER_NAME=$USER-karpenter-demo -AWS_DEFAULT_REGION=us-west-2 +KARPENTER_VERSION=$(curl -fsSL \ + https://api.github.com/repos/awslabs/karpenter/releases/latest \ + | jq -r '.tag_name') ``` ### Create a Cluster -Create an EKS cluster +Karpenter can run anywhere, including on self-managed node groups, [managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html), or [AWS Fargate](https://aws.amazon.com/fargate/). +This demo will run Karpenter on Fargate, which means all EC2 instances added to this cluster will be controlled by Karpenter. + ```bash -eksctl create cluster \ ---name ${CLUSTER_NAME} \ ---node-type m5.large \ ---nodes 1 \ ---nodes-min 1 \ ---nodes-max 10 \ ---managed \ ---with-oidc +curl -fsSL https://raw.githubusercontent.com/awslabs/karpenter/"${KARPENTER_VERSION}"/docs/aws/eks-config.yaml \ + | envsubst \ + | eksctl create cluster -f - ``` Tag the cluster subnets with the required tags for Karpenter auto discovery. -Note: If you have a cluster with version 1.18 or below you can skip this step. +> If you are using a cluster with version 1.18 or below you can skip this step. More [detailed here](https://github.com/awslabs/karpenter/issues/404#issuecomment-845283904). ```bash @@ -37,7 +38,7 @@ SUBNET_IDS=$(aws cloudformation describe-stacks \ --output text) aws ec2 create-tags \ - --resources $(echo $SUBNET_IDS | tr ',' '\n') \ + --resources $(echo ${SUBNET_IDS//,/ }) \ --tags Key="kubernetes.io/cluster/${CLUSTER_NAME}",Value= ``` @@ -45,46 +46,39 @@ aws ec2 create-tags \ We recommend using [CloudFormation](https://aws.amazon.com/cloudformation/) and [IAM Roles for Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) (IRSA) to manage these permissions. For production use, please review and restrict these permissions for your use case. -Note: For IRSA to work your [cluster needs an OIDC provider](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) +> For IRSA to work your cluster needs an [OIDC provider](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) ```bash -OIDC_PROVIDER=$(aws eks describe-cluster \ - --name ${CLUSTER_NAME} \ - --query 'cluster.identity.oidc.issuer' \ - --output text) - # Creates IAM resources used by Karpenter -LATEST_KARPENTER_VERSION=$(curl \ - https://api.github.com/repos/awslabs/karpenter/releases/latest | jq -r '.tag_name') TEMPOUT=$(mktemp) -curl -fsSL https://raw.githubusercontent.com/awslabs/karpenter/"${LATEST_KARPENTER_VERSION}"/docs/aws/karpenter.cloudformation.yaml > $TEMPOUT \ +curl -fsSL https://raw.githubusercontent.com/awslabs/karpenter/"${KARPENTER_VERSION}"/docs/aws/karpenter.cloudformation.yaml > $TEMPOUT \ && aws cloudformation deploy \ --stack-name Karpenter-${CLUSTER_NAME} \ --template-file ${TEMPOUT} \ --capabilities CAPABILITY_NAMED_IAM \ - --parameter-overrides ClusterName=${CLUSTER_NAME} OpenIDConnectIdentityProvider=${OIDC_PROVIDER/https:\/\//} - -# Adds the karpenter node role to your aws-auth configmap, allowing nodes with this role to connect to the cluster. -kubectl patch configmap aws-auth -n kube-system --patch "$(cat <<-EOM -data: - mapRoles: | - - rolearn: arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME} - username: system:node:{{EC2PrivateDNSName}} - groups: - - system:bootstrappers - - system:nodes -$(kubectl get configmap -n kube-system aws-auth -ojsonpath='{.data.mapRoles}' | sed 's/^/ /') -EOM -)" + --parameter-overrides ClusterName=${CLUSTER_NAME} + +# Add the karpenter node role to your aws-auth configmap, allowing nodes with this role to connect to the cluster. +eksctl create iamidentitymapping \ + --username system:node:{{EC2PrivateDNSName}} \ + --cluster ${CLUSTER_NAME} \ + --arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME} \ + --group system:bootstrappers \ + --group system:nodes ``` ### Install Karpenter + +Use [`helm`](https://helm.sh/) to deploy Karpenter to the cluster. +For additional values, see [the helm chart values](https://github.com/awslabs/karpenter/blob/main/charts/karpenter/values.yaml) + +> We created a Kubernetes service account with our cluster so we don't need the helm chart to do that. + ```bash helm repo add karpenter https://awslabs.github.io/karpenter/charts helm repo update -# For additional values, see https://github.com/awslabs/karpenter/blob/main/charts/karpenter/values.yaml -helm upgrade --install karpenter karpenter/karpenter --create-namespace --namespace karpenter \ - --set serviceAccount.annotations.'eks\.amazonaws\.com/role-arn'=arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterControllerRole-${CLUSTER_NAME} +helm upgrade --install karpenter karpenter/karpenter \ + --namespace karpenter --set serviceAccount.create=false ``` ### (Optional) Enable Verbose Logging @@ -103,17 +97,18 @@ kind: Provisioner metadata: name: default spec: + ttlSeconds: 30 cluster: name: ${CLUSTER_NAME} caBundle: $(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.certificateAuthority.data" --output json) endpoint: $(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output json) EOF -kubectl get provisioner default -oyaml +kubectl get provisioner default -o yaml ``` ### Create some pods Create some dummy pods and observe logs. -> Note: this will cause EC2 Instances to launch, which will be billed to your AWS Account. + ```bash cat < Karpenter automatically adds a node finalizer to properly cordon and drain nodes before they are terminated. +```bash +kubectl delete node $NODE_NAME +``` + ### Cleanup +> To avoid additional costs make sure you delete all ec2 instances before deleting the other cluster resources. ```bash helm delete karpenter -n karpenter aws cloudformation delete-stack --stack-name Karpenter-${CLUSTER_NAME} aws ec2 describe-launch-templates \ | jq -r ".LaunchTemplates[].LaunchTemplateName" \ - | grep -i karpenter \ + | grep -i Karpenter-${CLUSTER_NAME} \ | xargs -I{} aws ec2 delete-launch-template --launch-template-name {} ``` diff --git a/docs/aws/eks-config.yaml b/docs/aws/eks-config.yaml new file mode 100644 index 000000000000..fba429b22b2a --- /dev/null +++ b/docs/aws/eks-config.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig +metadata: + name: ${CLUSTER_NAME} + region: ${AWS_DEFAULT_REGION} + +iam: + withOIDC: true + serviceAccounts: + - metadata: + name: karpenter + namespace: karpenter + attachPolicy: + Version: "2012-10-17" + Statement: + - Effect: Allow + Resource: "*" + Action: + # Write Operations + - "ec2:CreateLaunchTemplate" + - "ec2:CreateFleet" + - "ec2:RunInstances" + - "ec2:CreateTags" + - "iam:PassRole" + - "ec2:TerminateInstances" + # Read Operations + - "ec2:DescribeLaunchTemplates" + - "ec2:DescribeInstances" + - "ec2:DescribeSecurityGroups" + - "ec2:DescribeSubnets" + - "ec2:DescribeInstanceTypes" + - "ec2:DescribeInstanceTypeOfferings" + - "ec2:DescribeAvailabilityZones" + - "ssm:GetParameter" + +fargateProfiles: + - name: karpenter + selectors: + - namespace: karpenter + - name: kube-system + selectors: + - namespace: kube-system diff --git a/docs/aws/karpenter.cloudformation.yaml b/docs/aws/karpenter.cloudformation.yaml index b46710312688..66d681d37678 100644 --- a/docs/aws/karpenter.cloudformation.yaml +++ b/docs/aws/karpenter.cloudformation.yaml @@ -1,64 +1,10 @@ AWSTemplateFormatVersion: "2010-09-09" Description: Resources used by https://github.com/awslabs/karpenter Parameters: - OpenIDConnectIdentityProvider: - Type: String - Description: "Example oidc.eks.us-west-2.amazonaws.com/id/1234567890" ClusterName: Type: String Description: "EKS cluster name" Resources: - KarpenterControllerRole: - Type: "AWS::IAM::Role" - Properties: - RoleName: !Sub "KarpenterControllerRole-${ClusterName}" - Path: / - AssumeRolePolicyDocument: !Sub | - { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Principal": { - "Federated": "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OpenIDConnectIdentityProvider}" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "${OpenIDConnectIdentityProvider}:aud": "sts.${AWS::URLSuffix}", - "${OpenIDConnectIdentityProvider}:sub": "system:serviceaccount:karpenter:karpenter" - } - } - }] - } - KarpenterControllerPolicy: - Type: "AWS::IAM::Policy" - Properties: - PolicyName: !Sub "KarpenterControllerPolicy-${ClusterName}" - Roles: - - - Ref: "KarpenterControllerRole" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Resource: "*" - Action: - # Write Operations - - "ec2:CreateLaunchTemplate" - - "ec2:CreateFleet" - - "ec2:RunInstances" - - "ec2:CreateTags" - - "iam:PassRole" - - "ec2:TerminateInstances" - # Read Operations - - "ec2:DescribeLaunchTemplates" - - "ec2:DescribeInstances" - - "ec2:DescribeSecurityGroups" - - "ec2:DescribeSubnets" - - "ec2:DescribeInstanceTypes" - - "ec2:DescribeInstanceTypeOfferings" - - "ec2:DescribeAvailabilityZones" - - "ssm:GetParameter" KarpenterNodeInstanceProfile: Type: "AWS::IAM::InstanceProfile" Properties: