add bootstrap functionality for kubelet on worker node

README.rst

@@ -82,6 +82,8 @@ To create the EKS cluster's control plane (master) with existing subnets of a VP
       --region us-west-2 \
       --kubconfig ./dev.conf \
       --heptio-auth /tmp/heptio-auth-aws \
+      --keyname dev \
+      --node-sg-ingress port=22,cidr=10.0.0.0/8 \
       --tags Env=dev,Project=eks-poc
 
 The simplest way to create a node group
@@ -107,6 +109,41 @@ To create a node group with more options
       --keyname dev \
       --tags Env=dev,Project=eks-poc
 
+To help bootstrapping kubelet agent
+
+.. code-block:: bash
+
+    # on EC2 worker instances, after copying kubelet, cni, heptio-aws-authenticator executables
+    $ eks bootstrap -o node-labels=gpu=enable,role=node \
+      -o feature-gates=RotateKubeletServerCertificate=true,CRIContainerLogRotation=true
+    $ systemctl daemon-reload
+    $ systemctl enable kubelet.service
+
+To display files created by ekscli boostrap locally rather than on EC2 instances
+
+.. code-block:: bash
+
+    # on EC2 instances
+    $ eks bootstrap --dry-run -n poc -r us-east-1 -m 32 \
+      -o node-labels=gpu=enable,role=node \
+      -o feature-gates=RotateKubeletServerCertificate=true,CRIContainerLogRotation=true
+
+To use ekscli boostrap as oneshot systemd unit
+
+.. code-block:: linux-config
+
+    [Unit]
+    Description=Configures Kubelet for EKS worker nodes
+    Before=kubelet.service
+
+    [Service]
+    Type=oneshot
+    ExecStart=/usr/local/bin/ekscli bootstrap
+    RemainAfterExit=true
+
+    [Install]
+    WantedBy=multi-user.target
+
 --------
 Features
 --------
@@ -116,6 +153,7 @@ Features
 * Plain vanilla EKS cluster without unrequired resources running Kubernetes clusters
 * EKS resources managed by AWS `CloudFormation <https://aws.amazon.com/cloudformation/>`_
 * Command line auto-completion supported for Bash and Zsh
+* Prepare necessary configuration for kubelet with self cluster discovery and additional options on worker nodes
 
 --------
 Roadmap

ekscli/__init__.py

@@ -4,7 +4,7 @@
 
 __author__ = """Charles Zhang"""
 __email__ = 'charles.cakab@gmail.com'
-__version__ = '0.1.0rc3'
+__version__ = '0.1.0rc5'
 __app_name__ = 'ekscli'
 
 

ekscli/cli.py

@@ -15,6 +15,7 @@
 from tabulate import tabulate
 
 import ekscli
+from ekscli.bootstrap import Kubelet
 from ekscli.stack import ControlPlane, KubeConfig, NodeGroup, ClusterInfo, AWSSecurityGroupRule
 from ekscli.thirdparty.click_alias import ClickAliasedGroup
 from ekscli.utils import which, MutuallyExclusiveOption
@@ -112,7 +113,8 @@ def validate_heptio_authenticator(ctx, param, value):
 def common_options(func):
     @click.option('--name', '-n', envvar='EKS_CLUSTER_NAME', required=True,
                   help='A regional unique name of the EKS cluster. Overrides EKS_CLUSTER_NAME environment variable.')
-    @click.option('--region', type=str, callback=validate_region, help='The AWS region to create the EKS cluster.')
+    @click.option('--region', '-r', type=str, callback=validate_region,
+                  help='The AWS region to create the EKS cluster.')
     @click.option('-v', '--verbosity', callback=config_logger, count=True,
                   help='Log level; -v for WARNING, -vv INFO,  -vvv DEBUG and -vvvv NOTSET.')
     @functools.wraps(func)
@@ -132,7 +134,7 @@ def eks(ctx):
 @eks.command()
 def version():
     """Show the EKS cli version info"""
-    print('Version '.format(ekscli.__version__))
+    print('Version {}'.format(ekscli.__version__))
 
 
 @eks.group(invoke_without_command=True, no_args_is_help=True, cls=ClickAliasedGroup,
@@ -164,6 +166,33 @@ def export(ctx):
     pass
 
 
+@eks.command()
+@click.option('--cluster-name', '-n', type=str, help='EKS cluster name')
+@click.option('--region', '-r', type=str, help='AWS region')
+@click.option('--max-pods', '-m', type=int, help='Max number pods able to run on the node.')
+@click.option('--node-ip', '-i', type=str, help='Node internal IP')
+@click.option('--kubelet-opt', '-o', type=str, multiple=True, help='kubelet options')
+@click.option('--kubelet-exec', '-e', type=str, help='kubelet executor file location', default='/usr/bin/kubelet',
+              show_default=True)
+@click.option('--kubelet-svc', '-s', type=str, help='kubelet service file location',
+              default='/etc/systemd/system/kubelet.service', show_default=True)
+@click.option('--kubeconf', '-k', type=str, help='kube-config file location', default='/var/lib/kubelet/kubeconfig',
+              show_default=True)
+@click.option('--cert', '-c', type=str, help='CA cert file location', default='/etc/kubernetes/pki/ca.crt',
+              show_default=True)
+@click.option('--dry-run', '-d', is_flag=True, default=False,
+              help='If true, only print the artifacts that could be written to files.')
+@click.pass_context
+def bootstrap(ctx, cluster_name, region, max_pods, node_ip, kubelet_opt, kubelet_exec, kubelet_svc, kubeconf, cert,
+              dry_run):
+    """Configure and bootstrap kubelet on worker nodes"""
+    opts = {v[0]: v[1] for v in [k if len(k) > 1 else k.append('') for k in [o.split('=', 1) for o in kubelet_opt]]}
+    kubelet = Kubelet(cluster_name=cluster_name, region=region, max_pods=max_pods, ip=node_ip, kubeconf_file=kubeconf,
+                      cert_file=cert, kubelet_opts=opts, kubelet_exec_file=kubelet_exec, kubelet_svc_file=kubelet_svc,
+                      dry_run=dry_run)
+    kubelet.bootstrap()
+
+
 @create.command(name='cluster')
 @common_options
 @click.option('--cp-role', type=str, help='The existing EKS role for the control plane.')
@@ -194,18 +223,25 @@ def export(ctx):
               mutex_group=['cp_only', 'node-max'], help='The max size of the node group')
 @click.option('--node-subnets', type=str, callback=validate_subnetes,
               help='The existing subnets to create node groups. Default, all subnets where EKS cluster is deployed.')
+@click.option('--node-type', type=str, cls=MutuallyExclusiveOption, mutex_group=['cp_only', 'node-type'],
+              help='Node group instance type.')
 @click.option('--keyname', type=str, cls=MutuallyExclusiveOption, mutex_group=['cp_only', 'keyname', 'ssh_public_key'],
               help='To use an existing keypair name in AWS for node groups')
 @click.option('--ssh-public-key', type=str, cls=MutuallyExclusiveOption,
               mutex_group=['cp_only', 'keyname', 'ssh_public_key'],
               help='To create a keypair used by node groups with an existing SSH public key.')
 @click.option('--ami', type=str, cls=MutuallyExclusiveOption, mutex_group=['cp_only', 'ami'],
               help='AWS AMI id or location')
+@click.option('--no-user-data', cls=MutuallyExclusiveOption, mutex_group=['cp_only', 'no_user_data'],
+              is_flag=True, default=False,
+              help='Not use the user data in NodeGroup LaunchConfiguration, '
+                   'instead ekscli-boostrap alike for node discovery.')
 @click.option('--yes', '-y', is_flag=True, default=False, help='Run ekscli without any confirmation prompt.')
 @click.pass_context
 def create_cluster(ctx, name, region, verbosity,
                    cp_role, subnets, tags, vpc_cidr, zones, kubeconf, username, heptio_auth, cp_only, node_name,
-                   node_role, node_sg_ingress, node_min, node_max, node_subnets, keyname, ssh_public_key, ami, yes):
+                   node_role, node_sg_ingress, node_min, node_max, node_subnets, node_type, keyname, ssh_public_key,
+                   ami, no_user_data, yes):
     """Create an EKS cluster"""
     if node_subnets and not subnets:
         print('If node subnets are specified, the cluster subnets must appear!')
@@ -236,7 +272,8 @@ def create_cluster(ctx, name, region, verbosity,
 
     ng = NodeGroup(node_name, cluster_info=cluster_info, keypair=keyname, region=region, ami=ami, subnets=node_subnets,
                    kubeconf=kubeconf, role=node_role, sg_ingresses=node_sg_ingress, min_nodes=node_min,
-                   max_nodes=node_max, ssh_public_key=ssh_public_key)
+                   max_nodes=node_max, instance_type=node_type, ssh_public_key=ssh_public_key,
+                   no_user_data=no_user_data)
     ng.create()
 
 
@@ -268,18 +305,25 @@ def export_kubeconfig(ctx, name, region, verbosity, kubeconf, username, heptio_a
               help='Additional security group ingresses for the node group')
 @click.option('--node-min', type=int, default=1, help='The min size of the node group')
 @click.option('--node-max', type=int, default=3, help='The max size of the node group')
+@click.option('--node-type', type=str, help='Node group instance type.')
+@click.option('--node-role', type=str, help='Additional roles for the node group')
 @click.option('--node-subnets', type=str, callback=validate_subnetes,
-              help='The existing subnets to create this node groups.')
+              help='The existing subnets to create node groups. Default, all subnets where EKS cluster is deployed.')
 @click.option('--keyname', type=str, help='To use an existing keypair name in AWS for node groups',
               cls=MutuallyExclusiveOption, mutex_group=['keyname', 'ssh_public_key'])
 @click.option('--ssh-public-key', type=str,
               help='To create a keypair used by node groups with an existing SSH public key.',
               cls=MutuallyExclusiveOption, mutex_group=['keyname', 'ssh_public_key'])
 @click.option('--ami', type=str, help='AWS AMI id or location')
+@click.option('--bootstrap-opt', '-b', type=str, multiple=True,
+              help='Options for ekscli bootstrap. See ekscli bootstrap --help.')
+@click.option('--no-user-data', type=str, is_flag=True, default=False,
+              help='Not use the user data in NodeGroup LaunchConfiguration, instead ekstrap alike for node discovery.')
 @click.option('--yes', '-y', is_flag=True, default=False, help='Run ekscli without any confirmation prompt.')
 @click.pass_context
 def create_nodegroup(ctx, name, node_name, region, verbosity, node_subnets, tags, kubeconf, node_min, node_max,
-                     node_role, node_sg_ingress, keyname, ssh_public_key, ami, yes):
+                     node_role, node_type, node_sg_ingress, keyname, ssh_public_key, ami, bootstrap_opt, no_user_data,
+                     yes):
     """Create a node group in an existing EKS cluster"""
     cp = ControlPlane(name, region=region)
     cluster_info = cp.query()
@@ -293,7 +337,8 @@ def create_nodegroup(ctx, name, node_name, region, verbosity, node_subnets, tags
                 exit(0)
     ng = NodeGroup(node_name, cluster_info=cluster_info, region=region, ami=ami, keypair=keyname, subnets=node_subnets,
                    role=node_role, sg_ingresses=node_sg_ingress, ssh_public_key=ssh_public_key, tags=tags,
-                   kubeconf=kubeconf, min_nodes=node_min, max_nodes=node_max)
+                   kubeconf=kubeconf, min_nodes=node_min, max_nodes=node_max, instance_type=node_type,
+                   no_user_data=no_user_data)
     ng.create()
 
 

ekscli/stack.py

@@ -4,6 +4,7 @@
 import logging
 import os
 import random
+import re
 import string
 import textwrap
 from collections import OrderedDict
@@ -104,6 +105,7 @@ class ControlPlane:
     OUTPUT_VPC = RESOURCE_EKS_VPC.name
     OUTPUT_SUBNETS = 'EKSSubnets'
 
+    CLUSTER_TAG_PATTERN = re.compile(r'kubernetes.io/cluster/(.*)')
     CLUSTER_TAG = 'kubernetes.io/cluster/{}'
 
     def __init__(self, name, role=None, subnets=None, region=None, kube_ver=None, tags=[],
@@ -523,7 +525,7 @@ class NodeGroup:
 
     def __init__(self, name, cluster_info=None, region=None, subnets=[], tags={}, min_nodes=1, max_nodes=3,
                  role=None, sg_ingresses=[], desired_nodes=1, ami=None, instance_type='m4.large', ssh_public_key=None,
-                 keypair=None, kubeconf=None):
+                 keypair=None, kubeconf=None, no_user_data=False):
         self.cluster = cluster_info
         self.subnets = subnets or self.cluster.subnets if self.cluster else []
         self.name = name
@@ -549,6 +551,7 @@ def __init__(self, name, cluster_info=None, region=None, subnets=[], tags={}, mi
         self.use_public_ip = False
         self.kubeconf = kubeconf
         self.resources = None
+        self.no_user_data = no_user_data
 
     def create(self):
         self._validate_creation()
@@ -646,7 +649,7 @@ def _validate_creation(self):
             images = list(ec2.images.filter(Owners=[owner], Filters=[{'Name': 'name', 'Values': [name]}]))
             if not images:
                 raise EKSCliException('image [{}] does not exist.'.format(self.ami))
-            self.ami = images[0]
+            self.ami = images[0].id
         else:
             image = ec2.Image(self.ami)
             if not image:
@@ -670,7 +673,6 @@ def _create_cfn_template(self):
 
         eks_tag = 'kubernetes.io/cluster/{}'.format(self.cluster.name)
 
-        # r = copy(self.RESOURCE_NG_ROLE)
         r = self.resources.get(self.RESOURCE_NG_ROLE.name)
         if self.role:
             profile = InstanceProfile(
@@ -698,7 +700,6 @@ def _create_cfn_template(self):
                 Output(self.RESOURCE_NG_ROLE.name, Value=GetAtt(role, 'Arn'), Description='Node group role'))
 
         self.tpl.add_resource(profile)
-        # self.resources.extend([r, copy(self.RESOURCE_NG_PROFILE)])
 
         if self.sg_igresses:
             sg = SecurityGroup(
@@ -752,20 +753,26 @@ def _create_cfn_template(self):
             r.status = Status.provided
 
         r.resource_id = self.keypair
-        # self.resources.insert(0, r)
 
         # auto-scaling group and launch configuration
-        userdata = [line + '\n' for line in
-                    Environment().from_string(self.USER_DATA).render(
-                        ci=self.cluster, ng_asg=self.RESOURCE_NG_ASG.name, stack_name=self.stack_name,
-                        max_pods=self.MAX_PODS.get(self.instance), region=self.region).split('\n')]
-
-        lc = LaunchConfiguration(
-            self.RESOURCE_NG_ASG_LC.name, AssociatePublicIpAddress=self.use_public_ip, IamInstanceProfile=Ref(profile),
-            ImageId=self.ami, InstanceType=self.instance, KeyName=self.keypair, SecurityGroups=[Ref(sg)],
-            UserData=Base64(Join('', userdata)))
-        self.tpl.add_resource(lc)
+        if self.no_user_data:
+            lc = LaunchConfiguration(
+                self.RESOURCE_NG_ASG_LC.name, AssociatePublicIpAddress=self.use_public_ip,
+                IamInstanceProfile=Ref(profile),
+                ImageId=self.ami, InstanceType=self.instance, KeyName=self.keypair, SecurityGroups=[Ref(sg)])
+        else:
+            user_data = Base64(
+                Join('', [line + '\n' for line in
+                          Environment().from_string(self.USER_DATA).render(
+                              ci=self.cluster, ng_asg=self.RESOURCE_NG_ASG.name, stack_name=self.stack_name,
+                              max_pods=self.MAX_PODS.get(self.instance), region=self.region).split('\n')]))
+            lc = LaunchConfiguration(
+                self.RESOURCE_NG_ASG_LC.name, AssociatePublicIpAddress=self.use_public_ip,
+                IamInstanceProfile=Ref(profile),
+                ImageId=self.ami, InstanceType=self.instance, KeyName=self.keypair, SecurityGroups=[Ref(sg)],
+                UserData=user_data)
 
+        self.tpl.add_resource(lc)
         self.tpl.add_resource(AutoScalingGroup(
             self.RESOURCE_NG_ASG.name, DesiredCapacity=self.desired, MinSize=self.min, MaxSize=self.max,
             LaunchConfigurationName=Ref(lc), VPCZoneIdentifier=self.subnets,
@@ -789,7 +796,7 @@ def delete(self, stack=None):
             cf = boto3.session.Session().resource('cloudformation')
             stack = cf.Stack(self.stack_name)
 
-        odict = {o.get('OutputKey'): o.get('OutputValue') for o in stack.outputs}
+        odict = {o.get('OutputKey'): o.get('OutputValue') for o in stack.outputs or []}
         key_name = odict.get(self.OUTPUT_KEYNAME)
 
         self._init_resources()

requirements-dev.txt

@@ -12,3 +12,4 @@ twine==1.10.0
 moto==1.3.3
 pytest==3.6.1
 pytest-runner==2.11.1
+configparser==3.5.0

requirements.txt

@@ -3,10 +3,11 @@ PyYAML==3.12
 click==6.7
 troposphere==2.3.1
 awacs==0.7.2
-boto3==1.7.37
+boto3==1.7.57
 halo==0.0.12
 kubernetes==6.0.0
 oyaml==0.4
 jinja2==2.10
 netaddr==0.7.19
 tabulate==0.8.2
+ec2-metadata==1.6.0

setup.py

@@ -23,7 +23,8 @@
                 'oyaml>=0.4',
                 'jinja2>=2.10',
                 'netaddr>=0.7.19',
-                'tabulate>=0.8.2'
+                'tabulate>=0.8.2',
+                'ec2-metadata>=1.6.0'
                 ]
 
 setup_requirements = ['pytest-runner', ]