Merge pull request kubernetes-retired#141 from mumoshu/fix-node-regis…

…tration-for-spot-fleet-powered-workers Fix node registration issue for spot-fleet-powered worker nodes
HotelsDotCom · Dec 8, 2016 · 1a2a0d2 · 1a2a0d2
2 parents 3b93c1e + 33cfe61
commit 1a2a0d2
Show file tree

Hide file tree

Showing 3 changed files with 65 additions and 0 deletions.
diff --git a/config/config.go b/config/config.go
@@ -281,6 +281,7 @@ type Cluster struct {
 	TLSCertDurationDays    int    `yaml:"tlsCertDurationDays,omitempty"`
 	HostedZone             string `yaml:"hostedZone,omitempty"`
 	HostedZoneID           string `yaml:"hostedZoneId,omitempty"`
+	Worker                 Worker
 	providedEncryptService EncryptService
 }
 
@@ -290,6 +291,17 @@ type Subnet struct {
 	lastAllocatedAddr *net.IP
 }
 
+// Just a place-holder to keep compatibility of cloud-config-worker between main cluster and node pool
+// Without this, {{if .Worker.SpotFleet.Enabled}} in the cloud-config-worker template fails with an obvious error like
+// "executing "CloudConfigWorker" at <.Worker>: can't evaluate field Worker in type *config.Config"
+type Worker struct {
+	SpotFleet SpotFleet
+}
+
+type SpotFleet struct {
+	Enabled bool
+}
+
 type Experimental struct {
 	AuditLog              AuditLog              `yaml:"auditLog"`
 	AwsEnvironment        AwsEnvironment        `yaml:"awsEnvironment"`

diff --git a/config/templates/cloud-config-worker b/config/templates/cloud-config-worker
@@ -211,6 +211,22 @@ coreos:
               --stack {{.ClusterName}}
 {{end}}
 
+{{if .Worker.SpotFleet.Enabled}}
+    - name: tag-spot-instance.service
+      enable: true
+      command: start
+      runtime: true
+      content: |
+        [Unit]
+        Description=Tag this spot instance with cluster name
+        After=network-online.target
+
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        ExecStart=/opt/bin/tag-spot-instance
+{{end}}
+
 {{ if $.ElasticFileSystemID }}
     - name: rpc-statd.service
       command: start
@@ -437,6 +453,36 @@ write_files:
       done
       echo done.
 
+{{if .Worker.SpotFleet.Enabled}}
+  - path: /opt/bin/tag-spot-instance
+    owner: root:root
+    permissions: 0700
+    content: |
+      #!/bin/bash -e
+
+      instance_id=$(curl http://169.254.169.254/latest/meta-data/instance-id)
+
+      sudo rkt run \
+        --volume=ssl,kind=host,source=/etc/kubernetes/ssl,readOnly=false \
+        --mount=volume=ssl,target=/etc/kubernetes/ssl \
+        --uuid-file-save=/var/run/coreos/tag-spot-instance.uuid \
+        --volume=dns,kind=host,source=/etc/resolv.conf,readOnly=true --mount volume=dns,target=/etc/resolv.conf \
+        --net=host \
+        --trust-keys-from-https \
+        --insecure-options=ondisk \
+        {{.AWSCliImageRepo}}:{{.AWSCliTag}} --exec=/bin/bash -- \
+          -vxc \
+          'echo tagging this spot instance; \
+           instance_id="'$instance_id'"; \
+           /usr/bin/aws \
+             --region {{.Region}} ec2 create-tags \
+             --resource $instance_id \
+             --tags "Key=KubernetesCluster,Value={{.ClusterName}}"; \
+           echo done.'
+
+      sudo rkt rm --uuid-file=/var/run/coreos/tag-spot-instance.uuid
+{{end}}
+
   - path: /opt/bin/taint-and-uncordon
     owner: root:root
     permissions: 0700

diff --git a/nodepool/config/templates/stack-template.json b/nodepool/config/templates/stack-template.json
@@ -280,6 +280,13 @@
                   "Resource": [ "*" ]
                 },
                 {{end}}
+                {{if .Worker.SpotFleet.Enabled}}
+                {
+                  "Action": "ec2:CreateTags",
+                  "Effect": "Allow",
+                  "Resource": "*"
+                },
+                {{end}}
                 {
                   "Action": [
                     "ecr:GetAuthorizationToken",