Attach EBS volumes to etcd nodes for persistence

README.md

@@ -49,6 +49,8 @@ creation
 * Bastion Host
 * Multi-AZ Auto-Scaling Worker Nodes
 * [NAT Gateway](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/vpc-nat-gateway.html)
+* Cluster state persisted using EBS
+* Automatic snapshotting of all EBS volumes, including dynamically generated persistent volumes
 
 ### CoreOS (1122.3.0, 1185.2.0, 1192.2.0)
 * etcd DNS Discovery Bootstrap
@@ -107,6 +109,7 @@ Terraform v0.7.7
 - Route 53 internal zone for VPC
 - Etcd cluster bootstrapped from Route 53
 - High Availability Kubernetes configuration (masters running on etcd nodes)
+- EBS volumes for etcd cluster with automatic snapshots
 - Autoscaling worker node group across subnets in selected region
 - kube-system namespace and addons: DNS, UI, Dashboard
 

modules.tf

@@ -122,6 +122,15 @@ module "worker" {
   worker-name = "general"
 }
 
+module "snapshot" {
+  source = "./modules/snapshot"
+
+  iam-role-snapshot-arn = "${ module.iam.iam-role-snapshot-arn }"
+  name = "${ var.name }"
+  security-groups = "${ module.security.etcd-id },${ module.security.worker-id }"
+  subnet-ids = "${ module.vpc.subnet-ids-private },${ module.vpc.subnet-ids-public }"
+}
+
 /*
 module "worker2" {
   source = "./modules/worker"

modules/etcd/cloud-config.tf

@@ -11,6 +11,7 @@ coreos:
     advertise-client-urls: http://${ fqdn }:2379
     # cert-file: /etc/kubernetes/ssl/k8s-etcd.pem
     # debug: true
+    data-dir: /media/etcd2
     discovery-srv: ${ internal-tld }
     initial-advertise-peer-urls: https://${ fqdn }:2380
     initial-cluster-state: new
@@ -25,6 +26,43 @@ coreos:
     peer-key-file: /etc/kubernetes/ssl/k8s-etcd-key.pem
 
   units:
+    - name: format-ebs-volume.service
+      command: start
+      content: |
+        [Unit]
+        Description=Formats the ebs volume
+        After=dev-xvdf.device
+        Requires=dev-xvdf.device
+        [Service]
+        ExecStart=/bin/bash -c "(/usr/sbin/blkid -t TYPE=ext4 | grep /dev/xvdf) || (/usr/sbin/wipefs -fa /dev/xvdf && /usr/sbin/mkfs.ext4 /dev/xvdf)"
+        RemainAfterExit=yes
+        Type=oneshot
+
+    - name: media-etcd2.mount
+      command: start
+      content: |
+        [Unit]
+        Description=Mount ebs to /media/etcd2
+        Requires=format-ebs-volume.service
+        After=format-ebs-volume.service
+        [Mount]
+        What=/dev/xvdf
+        Where=/media/etcd2
+        Type=ext4
+
+    - name: prepare-etcd-data-dir.service
+      command: start
+      content: |
+        [Unit]
+        Description=Prepares the etcd data directory
+        Requires=media-etcd2.mount
+        After=media-etcd2.mount
+        Before=etcd2.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStart=/usr/bin/chown -R etcd:etcd /media/etcd2
+
     - name: etcd2.service
       command: start
       drop-ins:

modules/etcd/ec2.tf

@@ -1,3 +1,30 @@
+resource "aws_ebs_volume" "etcd" {
+  count = "${ length( split(",", var.azs) ) }"
+
+  availability_zone = "${ element( split(",", var.azs), 0 ) }"
+
+  type = "gp2"
+  size = 100
+
+  tags {
+    builtWith = "terraform"
+    Cluster = "${ var.name }"
+    depends-id = "${ var.depends-id }"
+    KubernetesCluster = "${ var.name }"
+    Name = "etcd${ count.index + 1 }-${ var.name }"
+    role = "etcd,apiserver"
+    version = "${ var.coreos-hyperkube-tag }"
+  }
+}
+
+resource "aws_volume_attachment" "etcd" {
+  count = "${ length( split(",", var.azs) ) }"
+
+  device_name = "/dev/xvdf"
+  volume_id = "${ element(aws_ebs_volume.etcd.*.id, count.index) }"
+  instance_id = "${ element(aws_instance.etcd.*.id, count.index) }"
+}
+
 resource "aws_instance" "etcd" {
   count = "${ length( split(",", var.etcd-ips) ) }"
 
@@ -32,5 +59,9 @@ resource "aws_instance" "etcd" {
 }
 
 resource "null_resource" "dummy_dependency" {
-  depends_on = [ "aws_instance.etcd" ]
+  depends_on = [
+    "aws_instance.etcd",
+    "aws_ebs_volume.etcd",
+    "aws_volume_attachment.etcd"
+  ]
 }

modules/iam/io.tf

@@ -5,5 +5,6 @@ variable "name" {}
 output "depends-id" { value = "${ null_resource.dummy_dependency.id }" }
 output "aws-iam-role-etcd-id" { value = "${ aws_iam_role.master.id }" }
 output "aws-iam-role-worker-id" { value = "${ aws_iam_role.worker.id }" }
+output "iam-role-snapshot-arn" { value = "${ aws_iam_role.snapshot.arn }" }
 output "instance-profile-name-master" { value = "${ aws_iam_instance_profile.master.name }" }
 output "instance-profile-name-worker" { value = "${ aws_iam_instance_profile.worker.name }" }

modules/iam/snapshot.tf

@@ -0,0 +1,62 @@
+resource "aws_iam_role" "snapshot" {
+    name = "snapshot"
+    assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "lambda.amazonaws.com"
+      },
+      "Effect": "Allow",
+      "Sid": ""
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy" "snapshot" {
+  name = "snapshot-k8s-${ var.name }"
+
+  policy = <<EOS
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["logs:*"],
+      "Resource": "arn:aws:logs:*:*:*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:Describe*",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:CreateNetworkInterface",
+        "ec2:DescribeNetworkInterfaces",
+        "ec2:DetachNetworkInterface",
+        "ec2:DeleteNetworkInterface"
+      ],
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:CreateSnapshot",
+        "ec2:CreateTags",
+        "ec2:ModifySnapshotAttribute",
+        "ec2:ResetSnapshotAttribute"
+      ],
+      "Resource": ["*"]
+    }
+  ]
+}
+EOS
+
+  role = "${ aws_iam_role.snapshot.id }"
+}

modules/snapshot/cloudwatch.tf

@@ -0,0 +1,11 @@
+resource "aws_cloudwatch_event_rule" "snapshot" {
+  name = "snapshot-${ var.name }"
+  description = "Schedule snapshots of ebs volumes"
+
+  schedule_expression = "rate(2 hours)"
+}
+
+resource "aws_cloudwatch_event_target" "snapshot" {
+  rule = "${ aws_cloudwatch_event_rule.snapshot.name }"
+  arn = "${ aws_lambda_function.snapshot.arn }"
+}

modules/snapshot/io.tf

@@ -0,0 +1,4 @@
+variable "iam-role-snapshot-arn" {}
+variable "name" {}
+variable "security-groups" {}
+variable "subnet-ids" {}

modules/snapshot/lambda.tf

@@ -0,0 +1,37 @@
+resource "aws_lambda_permission" "snapshot" {
+  statement_id = "AllowExecutionFromCloudWatchForSnapshots"
+  action = "lambda:InvokeFunction"
+  function_name = "${ aws_lambda_function.snapshot.arn }"
+  principal = "events.amazonaws.com"
+  source_arn = "${ aws_cloudwatch_event_rule.snapshot.arn }"
+}
+
+resource "aws_lambda_function" "snapshot" {
+  filename = "${ path.module }/../../tmp/snapshot.zip"
+  function_name = "snapshot-${ var.name }"
+  handler = "snapshot.snapshot"
+  runtime = "python2.7"
+  source_code_hash = "${ base64sha256(data.template_file.init.rendered) }"
+
+  role = "${ var.iam-role-snapshot-arn }"
+
+  vpc_config {
+    subnet_ids = ["${ split(",", var.subnet-ids) }"]
+    security_group_ids = ["${ split(",", var.security-groups) }"]
+  }
+}
+
+data "template_file" "init" {
+  template = "${ file("${ path.module }/snapshot.py.tpl") }"
+
+  vars {
+    name = "${ var.name }"
+  }
+}
+
+resource "archive_file" "init" {
+  type = "zip"
+  source_content_filename = "snapshot.py"
+  source_content = "${ data.template_file.init.rendered }"
+  output_path = "${ path.module }/../../tmp/snapshot.zip"
+}

modules/snapshot/snapshot.py.tpl

@@ -0,0 +1,15 @@
+import boto3
+
+
+ec = boto3.client('ec2')
+
+def snapshot(event, context):
+    volumes = ec.describe_volumes(Filters=[{
+      'Name': 'tag:KubernetesCluster',
+      'Values': ['${ name }'], }])
+
+    for vol in volumes.get('Volumes', []):
+        snap = ec.create_snapshot(VolumeId=vol.get('VolumeId'))
+        ec.create_tags(
+            Resources=[snap.get('SnapshotId')],
+            Tags=vol.get('Tags'))