Merge pull request #682 from Hrishike/master

EFK stack using OpenEBS as the volume provisioner
Update README for running Kafka with OpenEBS
Initial commit for cockroachDB

Fixes #611 
Fixes #667

k8s/demo/cockroachDB/README.md

@@ -0,0 +1,5 @@
+Use CockroachDB with OpenEBS. 
+
+## TODO: 
+    - Install, Verification. 
+    - Insert Data, retrieve Data. 

k8s/demo/cockroachDB/cockroachdb-sts.yaml

@@ -0,0 +1,119 @@
+
+---
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  name: cockroachdb-budget
+  labels:
+    app: cockroachdb
+spec:
+  selector:
+    matchLabels:
+      app: cockroachdb
+  minAvailable: 67%
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: cockroachdb
+spec:
+  serviceName: "cockroachdb"
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: cockroachdb
+    spec:
+      # Init containers are run only once in the lifetime of a pod, before
+      # it's started up for the first time. It has to exit successfully
+      # before the pod's main containers are allowed to start.
+      # This particular init container does a DNS lookup for other pods in
+      # the set to help determine whether or not a cluster already exists.
+      # If any other pods exist, it creates a file in the cockroach-data
+      # directory to pass that information along to the primary container that
+      # has to decide what command-line flags to use when starting CockroachDB.
+      # This only matters when a pod's persistent volume is empty - if it has
+      # data from a previous execution, that data will always be used.
+      #
+      # If your Kubernetes cluster uses a custom DNS domain, you will have
+      # to add an additional arg to this pod: "-domain=<your-custom-domain>"
+      initContainers:
+      - name: bootstrap
+        image: cockroachdb/cockroach-k8s-init:0.2
+        imagePullPolicy: IfNotPresent
+        args:
+        - "-on-start=/on-start.sh"
+        - "-service=cockroachdb"
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - name: datadir
+          mountPath: /cockroach/cockroach-data
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                  - cockroachdb
+              topologyKey: kubernetes.io/hostname
+      containers:
+      - name: cockroachdb
+        image: cockroachdb/cockroach:v1.1.1
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 26257
+          name: grpc
+        - containerPort: 8080
+          name: http
+        volumeMounts:
+        - name: datadir
+          mountPath: /cockroach/cockroach-data
+        command:
+          - "/bin/bash"
+          - "-ecx"
+          - |
+            # The use of qualified `hostname -f` is crucial:
+            # Other nodes aren't able to look up the unqualified hostname.
+            CRARGS=("start" "--logtostderr" "--insecure" "--host" "$(hostname -f)" "--http-host" "0.0.0.0" "--cache" "25%" "--max-sql-memory" "25%")
+            # We only want to initialize a new cluster (by omitting the join flag)
+            # if we're sure that we're the first node (i.e. index 0) and that
+            # there aren't any other nodes running as part of the cluster that
+            # this is supposed to be a part of (which indicates that a cluster
+            # already exists and we should make sure not to create a new one).
+            # It's fine to run without --join on a restart if there aren't any
+            # other nodes.
+            if [ ! "$(hostname)" == "cockroachdb-0" ] || \
+               [ -e "/cockroach/cockroach-data/cluster_exists_marker" ]
+            then
+              # We don't join cockroachdb in order to avoid a node attempting
+              # to join itself, which currently doesn't work
+              # (https://github.com/cockroachdb/cockroach/issues/9625).
+              CRARGS+=("--join" "cockroachdb-public")
+            fi
+            exec /cockroach/cockroach ${CRARGS[*]}
+      # No pre-stop hook is required, a SIGTERM plus some time is all that's
+      # needed for graceful shutdown of a node.
+      terminationGracePeriodSeconds: 60
+      volumes:
+      - name: datadir
+        persistentVolumeClaim:
+          claimName: datadir
+  volumeClaimTemplates:
+  - metadata:
+      name: datadir
+      annotations:
+        volume.alpha.kubernetes.io/storage-class: openebs-cockroachdb-sc
+    spec:
+      accessModes:
+        - "ReadWriteOnce"
+      resources:
+        requests:
+          storage: 10G

k8s/demo/cockroachDB/cockroachdb-svc.yaml

@@ -0,0 +1,55 @@
+apiVersion: v1
+kind: Service
+metadata:
+  # This service is meant to be used by clients of the database. It exposes a ClusterIP that will
+  # automatically load balance connections to the different database pods.
+  name: cockroachdb-public
+  labels:
+    app: cockroachdb
+spec:
+  ports:
+  # The main port, served by gRPC, serves Postgres-flavor SQL, internode
+  # traffic and the cli.
+  - port: 26257
+    targetPort: 26257
+    name: grpc
+  # The secondary port serves the UI as well as health and debug endpoints.
+  - port: 8080
+    targetPort: 8080
+    name: http
+  selector:
+    app: cockroachdb
+---
+apiVersion: v1
+kind: Service
+metadata:
+  # This service only exists to create DNS entries for each pod in the stateful
+  # set such that they can resolve each other's IP addresses. It does not
+  # create a load-balanced ClusterIP and should not be used directly by clients
+  # in most circumstances.
+  name: cockroachdb
+  labels:
+    app: cockroachdb
+  annotations:
+    # This is needed to make the peer-finder work properly and to help avoid
+    # edge cases where instance 0 comes up after losing its data and needs to
+    # decide whether it should create a new cluster or try to join an existing
+    # one. If it creates a new cluster when it should have joined an existing
+    # one, we'd end up with two separate clusters listening at the same service
+    # endpoint, which would be very bad.
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+    # Enable automatic monitoring of all instances when Prometheus is running in the cluster.
+    prometheus.io/scrape: "true"
+    prometheus.io/path: "_status/vars"
+    prometheus.io/port: "8080"
+spec:
+  ports:
+  - port: 26257
+    targetPort: 26257
+    name: grpc
+  - port: 8080
+    targetPort: 8080
+    name: http
+  clusterIP: None
+  selector:
+    app: cockroachdb

k8s/demo/cockroachDB/openEBS-sc.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+   name: openebs-cockroachdb-sc
+provisioner: openebs.io/provisioner-iscsi
+parameters:
+  pool: hostdir-var
+  replica: "2"
+  size: 5G
+---

k8s/demo/efk/README.md

@@ -0,0 +1,87 @@
+Apply the specs in the efk folder.
+Make sure you have applied the Storage classes for OpenEBS.
+
+`kubectl apply -f openebs-sc.yaml`
+
+This EFK podspec uses Elasticsearch, Fluentd and Kibana to enable you to perform k8s cluster level logging. 
+The fluentd pods act as collectors, Elasticsearch as the document database and kibana as the dashboard for log visualization. 
+
+The current podspec for Elasticsearch creates
+    1) 3 master pods responsible for cluster management.
+    2) 3 data pods for storing log data.
+    3) 2 client pods for external access.
+
+The current Fluentd podspec reads journal logs for `kubelet` and cluster level logging by reading from `/var/log/containers` for pods running on the kubernetes cluster. 
+
+#### Note: Make sure you install Elasticsearch while executing this usecase. Fluentd and Kibana require the publicly accessible Elastic search endpoint. 
+
+## Verify Elastic search installation
+
+```
+curl 'http://10.105.105.41:9200'
+{
+  "name" : "es-client-2155074821-nxdkt",
+  "cluster_name" : "escluster",
+  "cluster_uuid" : "zAYA9ERGQgCEclvYHCsOsA",
+  "version" : {
+    "number" : "5.5.0",
+    "build_hash" : "260387d",
+    "build_date" : "2017-06-30T23:16:05.735Z",
+    "build_snapshot" : false,
+    "lucene_version" : "6.6.0"
+  },
+  "tagline" : "You Know, for Search"
+}
+
+curl 'http://10.105.105.41:9200/_cat/nodes?v'
+ip        heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
+10.44.0.2           41          41   0    0.00    0.03     0.08 m         -      es-master-2996564765-4c56v
+10.36.0.1           43          18   0    0.07    0.05     0.05 i         -      es-client-2155074821-v0w31
+10.40.0.2           49          15   0    0.05    0.07     0.11 m         *      es-master-2996564765-zj0gc
+10.47.0.3           43          20   0    0.13    0.11     0.13 i         -      es-client-2155074821-nxdkt
+10.47.0.4           42          20   0    0.13    0.11     0.13 d         -      elasticsearch-data-2
+10.47.0.2           39          20   0    0.13    0.11     0.13 m         -      es-master-2996564765-rql6m
+10.42.0.2           41          13   0    0.00    0.04     0.10 d         -      elasticsearch-data-1
+10.40.0.3           42          15   0    0.05    0.07     0.11 d         -      elasticsearch-data-0
+
+curl -XPUT 'http://10.105.105.41:9200/customer?pretty&pretty'
+{
+  "acknowledged" : true,
+  "shards_acknowledged" : true
+}
+
+curl -XGET 'http://10.105.105.41:9200/_cat/indices?v&pretty'
+health status index    uuid                   pri rep docs.count docs.deleted store.size pri.store.size
+green  open   customer -Cort549Sn6q4gmbwicOMA   5   1          0            0      1.5kb           810b
+
+curl -XPUT 'http://10.105.105.41:9200/customer/external/1?pretty&pretty' -H 'Content-Type: application/json' -d'
+{
+"name": "Daenerys Targaryen"
+}
+'
+{
+  "_index" : "customer",
+  "_type" : "external",
+  "_id" : "1",
+  "_version" : 1,
+  "result" : "created",
+  "_shards" : {
+    "total" : 2,
+    "successful" : 2,
+    "failed" : 0
+  },
+  "created" : true
+}
+
+curl 'http://10.105.105.41:9200/customer/external/1?pretty&pretty'
+{
+  "_index" : "customer",
+  "_type" : "external",
+  "_id" : "1",
+  "_version" : 1,
+  "found" : true,
+  "_source" : {
+    "name" : "Daenerys Targaryen"
+  }
+}
+```

k8s/demo/efk/es/es-client-rc.yaml

@@ -0,0 +1,65 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: es-client
+  labels:
+    component: elasticsearch
+    role: client
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        component: elasticsearch
+        role: client
+    spec:
+      initContainers:
+      - name: init-sysctl
+        image: busybox
+        imagePullPolicy: IfNotPresent
+        command: ["sysctl", "-w", "vm.max_map_count=262144"]
+        securityContext:
+          privileged: true
+      containers:
+      - name: es-client
+        securityContext:
+          privileged: false
+          capabilities:
+            add:
+              - IPC_LOCK
+              - SYS_RESOURCE
+        image: quay.io/pires/docker-elasticsearch-kubernetes:5.5.0
+        imagePullPolicy: Always
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: "CLUSTER_NAME"
+          value: "myesdb"
+        - name: NODE_MASTER
+          value: "false"
+        - name: NODE_DATA
+          value: "false"
+        - name: HTTP_ENABLE
+          value: "true"
+        - name: "ES_JAVA_OPTS"
+          value: "-Xms256m -Xmx256m"
+        ports:
+        - containerPort: 9200
+          name: http
+          protocol: TCP
+        - containerPort: 9300
+          name: transport
+          protocol: TCP
+        volumeMounts:
+        - name: storage
+          mountPath: /data
+      volumes:
+          - emptyDir:
+              medium: ""
+            name: "storage"

k8s/demo/efk/es/es-client-svc.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+  labels:
+    component: elasticsearch
+    role: client
+spec:
+  type: LoadBalancer
+  selector:
+    component: elasticsearch
+    role: client
+  ports:
+  - name: http
+    port: 9200
+    protocol: TCP