Add elasticsearch for Kubernetes logs

01-env.sh

@@ -45,6 +45,7 @@ export PATH="$(cd $(dirname -- "$0") && pwd)/.bin:$PATH"
 ## Terraform output used by Helmfile
 
 export efs_provisoner_file_system_id="$(terraform output efs_provisoner_file_system_id 2> /dev/null)"
+export es_logs_endpoint="$(terraform output es_logs_endpoint 2> /dev/null)"
 
 
 

README.md

@@ -104,12 +104,15 @@ By default the script will create the following components:
   - A Route53 record for the internet-facing ALB
   - A security group for the internet-facing ALB
   - An EFS filesystem for Persistent Volumes
+  - An Elasticsearch domain for Kubernetes logs
 - kubectl
   - Create `ServiceAccount` and `ClusterRoleBinding` for the Helm tiller
   - Patch `StorageClass/gp2` to remove the default storage class
 - Helm
   - `nginx-ingress`
   - `efs-provisioner`
+  - `fluent-bit`
+  - `kibana`
 
 Bootstrap a cluster.
 
@@ -265,7 +268,7 @@ Cluster | Route53 Hosted Zone | - | $0.5
 Cluster | S3  | - | free
 Managed | EFS | General Purpose up to 5GB | free
 Managed | RDS | t2.micro gp2 20GB | free
-Managed | Elasticsearch | t2.micro gp2 10GB | free
+Managed | Elasticsearch | t2.small gp2 10GB | free
 
 The cluster name must be a domain name in order to reduce an ELB for masters.
 

helmfile.yaml

@@ -48,3 +48,37 @@ releases:
           storageClass:
             name: efs
             isDefault: true
+
+  - name: fluent-bit
+    namespace: kube-system
+    chart: stable/fluent-bit
+    values:
+      # https://github.com/helm/charts/tree/master/stable/fluent-bit
+      - backend:
+          type: es
+          es:
+            host: {{ requiredEnv "es_logs_endpoint" }}
+            port: 443
+            tls: "on"
+        resources:
+          limits:
+            memory: 64Mi
+          requests:
+            cpu: 0
+            memory: 64Mi
+
+  - name: kibana
+    namespace: kube-system
+    chart: stable/kibana
+    values:
+      # https://github.com/helm/charts/tree/master/stable/kibana
+      - env:
+          ELASTICSEARCH_URL: https://{{ requiredEnv "es_logs_endpoint" }}:443
+        image:
+          tag: 6.2.4
+        # TODO: ingress
+        resources:
+          limits:
+            memory: 256Mi
+          requests:
+            memory: 256Mi

tf_config.tf

@@ -56,5 +56,5 @@ variable "database_admin_password" {
 
 locals {
   # Hash of kubernetes_cluster_name and kubernetes_ingress_domain
-  alb_name_hash = "${substr(sha256("${var.kubernetes_cluster_name}/${var.kubernetes_ingress_domain}"), 0, 16)}"
+  kubernetes_cluster_name_hash = "${substr(sha256("${var.kubernetes_cluster_name}/${var.kubernetes_ingress_domain}"), 0, 16)}"
 }

tf_external_alb.tf

@@ -1,7 +1,7 @@
 ## External ALB for Kubernetes services.
 
 resource "aws_lb" "alb_external" {
-  name               = "alb-ext-${local.alb_name_hash}"
+  name               = "alb-ext-${local.kubernetes_cluster_name_hash}"
   load_balancer_type = "application"
   internal           = false
   idle_timeout       = 180
@@ -55,7 +55,7 @@ resource "aws_lb_listener" "alb_external" {
 }
 
 resource "aws_lb_target_group" "alb_external" {
-  name                 = "alb-ext-${local.alb_name_hash}"
+  name                 = "alb-ext-${local.kubernetes_cluster_name_hash}"
   port                 = 30080
   protocol             = "HTTP"
   vpc_id               = "${data.aws_vpc.kops_vpc.id}"

tf_internal_alb.tf

@@ -4,7 +4,7 @@
 
 resource "aws_lb" "alb_internal" {
   count              = "${var.alb_internal_enabled}"
-  name               = "alb-int-${local.alb_name_hash}"
+  name               = "alb-int-${local.kubernetes_cluster_name_hash}"
   load_balancer_type = "application"
   internal           = true
   idle_timeout       = 180
@@ -67,7 +67,7 @@ resource "aws_lb_listener" "alb_internal" {
 
 resource "aws_lb_target_group" "alb_internal" {
   count                = "${var.alb_internal_enabled}"
-  name                 = "alb-int-${local.alb_name_hash}"
+  name                 = "alb-int-${local.kubernetes_cluster_name_hash}"
   port                 = 30080
   protocol             = "HTTP"
   vpc_id               = "${data.aws_vpc.kops_vpc.id}"

tf_managed_services.tf

@@ -41,6 +41,65 @@ output "efs_provisoner_file_system_id" {
   value = "${aws_efs_file_system.efs_provisioner.id}"
 }
 
+# Elasticsearch for Kubernetes logs
+resource "aws_elasticsearch_domain" "logs" {
+  domain_name           = "logs-${local.kubernetes_cluster_name_hash}"
+  elasticsearch_version = "6.2"
+
+  cluster_config {
+    instance_type          = "t2.small.elasticsearch"
+    zone_awareness_enabled = false
+  }
+
+  ebs_options {
+    ebs_enabled = true
+    volume_type = "gp2"
+    volume_size = 10
+  }
+
+  vpc_options {
+    subnet_ids         = ["${data.aws_subnet_ids.kops_subnets.ids[0]}"]
+    security_group_ids = ["${aws_security_group.allow_from_k8s_nodes.id}"]
+  }
+
+  tags = "${merge(
+    map("kubernetes.io/cluster/${var.kubernetes_cluster_name}", "owned"),
+    map("Name", "logs.${var.kubernetes_cluster_name}")
+  )}"
+}
+
+resource "aws_iam_service_linked_role" "es" {
+  # https://github.com/terraform-providers/terraform-provider-aws/issues/5218
+  aws_service_name = "es.amazonaws.com"
+}
+
+data "aws_iam_policy_document" "es_logs_access" {
+  statement {
+    actions = [
+      "es:*",
+    ]
+
+    resources = [
+      "${aws_elasticsearch_domain.logs.arn}",
+      "${aws_elasticsearch_domain.logs.arn}/*",
+    ]
+
+    principals {
+      type        = "AWS"
+      identifiers = ["*"]
+    }
+  }
+}
+
+resource "aws_elasticsearch_domain_policy" "es_logs_access" {
+  domain_name     = "${aws_elasticsearch_domain.logs.domain_name}"
+  access_policies = "${data.aws_iam_policy_document.es_logs_access.json}"
+}
+
+output "es_logs_endpoint" {
+  value = "${aws_elasticsearch_domain.logs.endpoint}"
+}
+
 # RDS
 resource "aws_db_subnet_group" "rds_for_k8s_nodes" {
   name       = "rds-for-nodes.${var.kubernetes_cluster_name}"