add configuration for deploy exchanges connector with metrics to gcloud

bro-n-bro · Apr 14, 2018 · aff86cf · aff86cf
1 parent 7132d84
commit aff86cf
Show file tree

Hide file tree

Showing 9 changed files with 378 additions and 3 deletions.
diff --git a/markets/common/elassandra.yaml b/markets/common/elassandra.yaml
@@ -0,0 +1,87 @@
+# Elassandra Service
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: elassandra
+  name: elassandra
+  namespace: markets
+spec:
+  clusterIP: None
+  ports:
+  - port: 9042
+  selector:
+    app: elassandra
+---
+# Elassandra StatefulSet
+apiVersion: apps/v1beta2
+kind: StatefulSet
+metadata:
+  name: elassandra
+  namespace: markets
+spec:
+  serviceName: elassandra
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elassandra
+  template:
+    metadata:
+      namespace: markets
+      labels:
+        app: elassandra
+    spec:
+      containers:
+      - name: elassandra
+        image: strapdata/elassandra:5.5.0.13
+        ports:
+        - containerPort: 7000
+          name: intra-node
+        - containerPort: 7001
+          name: tls-intra-node
+        - containerPort: 7199
+          name: jmx
+        - containerPort: 9042
+          name: cql
+        - containerPort: 9200
+          name: elastic-http
+        - containerPort: 9300
+          name: elastic-intra
+        resources:
+          requests:
+           memory: 10Gi
+        securityContext:
+          capabilities:
+            add:
+              - IPC_LOCK
+        lifecycle:
+          preStop:
+            exec:
+              command: ["/bin/sh", "-c", "PID=$(pidof java) && kill $PID && while ps -p $PID > /dev/null; do sleep 1; done"]
+        env:
+          - name: CASSANDRA_NUM_TOKENS
+            value: "16"
+          - name: CASSANDRA_SEEDS
+            value: "elassandra-0.elassandra.markets.svc.cluster.local"
+          - name: CASSANDRA_CLUSTER_NAME
+            value: "CYBER_MARKETS"
+          - name: CASSANDRA_DC
+            value: "PRODUCTION"
+          - name: CASSANDRA_AUTO_BOOTSTRAP
+            value: "true"
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+        volumeMounts:
+        - name: elassandra-data
+          mountPath: /var/lib/cassandra
+  volumeClaimTemplates:
+  - metadata:
+      name: elassandra-data
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 300Gi
+      storageClassName: fast
diff --git a/markets/common/kafka.yaml b/markets/common/kafka.yaml
@@ -0,0 +1,92 @@
+
+# Kafka Data Volume Claim
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: kafka-broker-claim0
+  namespace: markets
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 500Gi
+---
+
+# Kafka Service
+apiVersion: v1
+kind: Service
+metadata:
+  name: kafka-broker
+  namespace: markets
+spec:
+  ports:
+  - name: "transport"
+    port: 9092
+    targetPort: 9092
+  - name: "jmx"
+    port: 1099
+    targetPort: 1099
+  selector:
+    app: kafka-broker
+---
+
+# Kafka Deployment
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  namespace: markets
+  name: kafka-broker
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      namespace: markets
+      labels:
+        app: kafka-broker
+    spec:
+      containers:
+      - env:
+        - name: KAFKA_ADVERTISED_LISTENERS
+          value: PLAINTEXT://kafka-broker:9092
+        - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE
+          value: "true"
+        - name: KAFKA_BROKER_ID
+          value: "1"
+        - name: KAFKA_DEFAULT_REPLICATION_FACTOR
+          value: "1"
+        - name: KAFKA_DELETE_TOPIC_ENABLE
+          value: "true"
+        - name: KAFKA_JMX_PORT
+          value: "1099"
+        - name: KAFKA_LOG4J_ROOT_LOGLEVEL
+          value: WARN
+        - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
+          value: "1"
+        - name: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR
+          value: "1"
+        - name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
+          value: "1"
+        - name: KAFKA_ZOOKEEPER_CONNECT
+          value: zoo:2181
+        - name: KAFKA_MESSAGE_MAX_BYTES
+          value: "15728640"
+        - name: KAFKA_REPLICA_FETCH_MAX_BYTES
+          value: "15728640"
+        image: confluentinc/cp-kafka:4.0.0-3
+        name: kafka-broker
+        ports:
+        - containerPort: 9092
+        - containerPort: 1099
+        resources: {}
+        volumeMounts:
+        - mountPath: /var/lib/kafka/data
+          name: kafka-broker-claim0
+          subPath: data
+      restartPolicy: Always
+      volumes:
+      - name: kafka-broker-claim0
+        persistentVolumeClaim:
+          claimName: kafka-broker-claim0
diff --git a/markets/common/zoo.yaml b/markets/common/zoo.yaml
@@ -0,0 +1,83 @@
+
+# Zookeeper Log Volume Claim
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: zoo-claim0
+  namespace: markets
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 1Gi
+---
+
+# Zookeeper Data Volume Claim
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: zoo-claim1
+  namespace: markets
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 1Gi
+---
+
+# Zookeeper Service
+apiVersion: v1
+kind: Service
+metadata:
+  name: zoo
+  namespace: markets
+spec:
+  ports:
+  - name: "2181"
+    port: 2181
+    targetPort: 2181
+  selector:
+    app: zoo
+---
+
+# Zookeeper Deployment
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: zoo
+  namespace: markets
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      namespace: markets
+      labels:
+        app: zoo
+    spec:
+      containers:
+      - env:
+        - name: KAFKA_LOG4J_ROOT_LOGLEVEL
+          value: ERROR
+        - name: ZOOKEEPER_CLIENT_PORT
+          value: "2181"
+        - name: ZOOKEEPER_TICK_TIME
+          value: "2000"
+        image: confluentinc/cp-zookeeper:4.0.0-3
+        name: zoo
+        ports:
+        - containerPort: 2181
+        volumeMounts:
+        - mountPath: /var/lib/zookeeper/log
+          name: zoo-claim0
+        - mountPath: /var/lib/zookeeper/data
+          name: zoo-claim1
+      restartPolicy: Always
+      volumes:
+      - name: zoo-claim0
+        persistentVolumeClaim:
+          claimName: zoo-claim0
+      - name: zoo-claim1
+        persistentVolumeClaim:
+          claimName: zoo-claim1
diff --git a/markets/exchanges-connector-cex.yaml b/markets/exchanges-connector-cex.yaml
@@ -0,0 +1,47 @@
+# Exchanges Connector CEX Service (should expose metrics)
+apiVersion: v1
+kind: Service
+metadata:
+  name: exchanges-connector-cex
+  namespace: markets
+  labels:
+    monitoring: default-monitor
+spec:
+  ports:
+  - name: "8080"
+    port: 8080
+    targetPort: 8080
+  selector:
+    app: exchanges-connector-cex
+---
+
+# Exchanges Connector CEX Deployment
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  namespace: markets
+  name: exchanges-connector-cex
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      namespace: markets
+      labels:
+        app: exchanges-connector-cex
+    spec:
+      containers:
+      - env:
+        - name: CM_LOG_LEVEL
+          value: INFO
+        - name: KAFKA_BROKERS
+          value: kafka-broker:9092
+        - name: EXCHANGES
+          value: bitfinex,bitstamp,gdax,hitbtc,poloniex,binance
+        image: cybernode/cm-exchanges-connector:0.2.0
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8080
+        name: exchanges-connector-cex
+      restartPolicy: Always
diff --git a/markets/exchanges-connector-dex.yaml b/markets/exchanges-connector-dex.yaml
@@ -0,0 +1,49 @@
+# Exchanges Connector DEX (should expose metrics)
+apiVersion: v1
+kind: Service
+metadata:
+  name: exchanges-connector-dex
+  namespace: markets
+  labels:
+    monitoring: default-monitor
+spec:
+  ports:
+  - name: "8080"
+    port: 8080
+    targetPort: 8080
+  selector:
+    app: exchanges-connector-dex
+---
+
+# Exchanges Connector DEX Deployment
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  namespace: markets
+  name: exchanges-connector-dex
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      namespace: markets
+      labels:
+        app: exchanges-connector-dex
+    spec:
+      containers:
+      - env:
+        - name: PARITY_URL
+          value: http://parity-eth.chains.svc:8545
+        - name: CM_LOG_LEVEL
+          value: INFO
+        - name: KAFKA_BROKERS
+          value: kafka-broker:9092
+        - name: EXCHANGES
+          value: etherdelta
+        image: cybernode/cm-exchanges-connector:0.2.0
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8080
+        name: exchanges-connector-dex
+      restartPolicy: Always
diff --git a/monitoring/dashboards-configmap.sh b/monitoring/dashboards-configmap.sh
@@ -5,3 +5,6 @@ kubectl delete configmap grafana-dashboards-etc --namespace=monitoring
 kubectl create configmap grafana-dashboards-common --from-file=dashboards/ --namespace=monitoring
 kubectl create configmap grafana-dashboards-eth --from-file=dashboards/ethereum --namespace=monitoring
 kubectl create configmap grafana-dashboards-etc --from-file=dashboards/ethereum_classic --namespace=monitoring
+
+kubectl delete configmap grafana-dashboards-exchanges --namespace=monitoring
+kubectl create configmap grafana-dashboards-exchanges --from-file=dashboards/markets --namespace=monitoring