Skip to content
This repository has been archived by the owner on Sep 30, 2020. It is now read-only.

Fix node registration issue for spot-fleet-powered worker nodes #141

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
12 changes: 12 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,7 @@ type Cluster struct {
TLSCertDurationDays int `yaml:"tlsCertDurationDays,omitempty"`
HostedZone string `yaml:"hostedZone,omitempty"`
HostedZoneID string `yaml:"hostedZoneId,omitempty"`
Worker Worker
providedEncryptService EncryptService
}

Expand All @@ -290,6 +291,17 @@ type Subnet struct {
lastAllocatedAddr *net.IP
}

// Just a place-holder to keep compatibility of cloud-config-worker between main cluster and node pool
// Without this, {{if .Worker.SpotFleet.Enabled}} in the cloud-config-worker template fails with an obvious error like
// "executing "CloudConfigWorker" at <.Worker>: can't evaluate field Worker in type *config.Config"
type Worker struct {
SpotFleet SpotFleet
}

type SpotFleet struct {
Enabled bool
}

type Experimental struct {
AuditLog AuditLog `yaml:"auditLog"`
AwsEnvironment AwsEnvironment `yaml:"awsEnvironment"`
Expand Down
46 changes: 46 additions & 0 deletions config/templates/cloud-config-worker
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,22 @@ coreos:
--stack {{.ClusterName}}
{{end}}

{{if .Worker.SpotFleet.Enabled}}
- name: tag-spot-instance.service
enable: true
command: start
runtime: true
content: |
[Unit]
Description=Tag this spot instance with cluster name
After=network-online.target

[Service]
Type=oneshot
RemainAfterExit=true
ExecStart=/opt/bin/tag-spot-instance
{{end}}

{{ if $.ElasticFileSystemID }}
- name: rpc-statd.service
command: start
Expand Down Expand Up @@ -437,6 +453,36 @@ write_files:
done
echo done.

{{if .Worker.SpotFleet.Enabled}}
- path: /opt/bin/tag-spot-instance
owner: root:root
permissions: 0700
content: |
#!/bin/bash -e

instance_id=$(curl http://169.254.169.254/latest/meta-data/instance-id)

sudo rkt run \
--volume=ssl,kind=host,source=/etc/kubernetes/ssl,readOnly=false \
--mount=volume=ssl,target=/etc/kubernetes/ssl \
--uuid-file-save=/var/run/coreos/tag-spot-instance.uuid \
--volume=dns,kind=host,source=/etc/resolv.conf,readOnly=true --mount volume=dns,target=/etc/resolv.conf \
--net=host \
--trust-keys-from-https \
--insecure-options=ondisk \
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

--insecure-options=ondisk to skip verification on docker images on the disk.
The verification is IMHO not needed for us as we don't verify when running images using docker.
Skipping verification results in faster container startup.

{{.AWSCliImageRepo}}:{{.AWSCliTag}} --exec=/bin/bash -- \
-vxc \
'echo tagging this spot instance; \
instance_id="'$instance_id'"; \
/usr/bin/aws \
--region {{.Region}} ec2 create-tags \
--resource $instance_id \
--tags "Key=KubernetesCluster,Value={{.ClusterName}}"; \
echo done.'

sudo rkt rm --uuid-file=/var/run/coreos/tag-spot-instance.uuid
{{end}}

- path: /opt/bin/taint-and-uncordon
owner: root:root
permissions: 0700
Expand Down
7 changes: 7 additions & 0 deletions nodepool/config/templates/stack-template.json
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,13 @@
"Resource": [ "*" ]
},
{{end}}
{{if .Worker.SpotFleet.Enabled}}
{
"Action": "ec2:CreateTags",
"Effect": "Allow",
"Resource": "*"
},
{{end}}
{
"Action": [
"ecr:GetAuthorizationToken",
Expand Down