Adding kafka rest proxy deployment

ansible/datalake.yml

@@ -21,8 +21,10 @@
 ---
 - hosts: data-lake
   roles:
+    - env_setup
+    - kafka
 #    - custos
 #    - consul
 #    - mft
-    - data_lake
+#    - data_lake
 #    - workflow-engine

ansible/inventories/test/group_vars/all/vars.yml

@@ -30,6 +30,7 @@ datalake_deployment_dir: "{{ user_home }}/data-lake-deployment"
 datalake_source_dir: "{{ user_home }}/data-lake-source"
 datalake_repo: "https://github.com/apache/airavata-data-lake.git"
 datalake_git_branch: master
+kafka_url: 149.165.156.200:9092
 
 mft_deployment_dir: "{{ user_home }}/mft-deployment"
 mft_source_dir: "{{ user_home }}/mft-source"
@@ -84,7 +85,7 @@ datalake_drms_custos_host: custos.scigap.org
 datalake_drms_custos_port: 31499
 datalake_drms_http_port: 8900
 
-datalake_synch_broker_url: 149.165.156.200:9092
+datalake_synch_broker_url: "{{ kafka_url }}"
 datalake_synch_broker_consumer_group: custosEventsGroup
 datalake_synch_broker_topic: 10002640-561b67b4-fa7d-4368-8643-dcd9d0159c1d
 datalake_synch_tenant_id: custos-ii8g0cfwsz6ruwezykn9-10002640
@@ -94,8 +95,8 @@ datalake_data_orch_datasource_username: "{{ vault_datalake_data_orch_datasource_
 datalake_data_orch_datasource_password: "{{ vault_datalake_data_orch_datasource_password }}"
 datalake_data_orch_http_port: 8899
 datalake_data_orch_grpc_port: 6566
-datalake_data_orch_broker_url: localhost:9092
-datalake_data_orch_broker_consumer_group: DataOrchestratorGroup3
+datalake_data_orch_broker_url: "{{ kafka_url }}"
+datalake_data_orch_broker_consumer_group: DataOrchestratorGroup2
 datalake_data_orch_broker_topic: data-orchestrator-file-events
 
 workflow_deployment_dir: "{{ user_home }}/workflow-deployment"
@@ -105,7 +106,7 @@ workflow_helix_cluster_name: datalake
 workflow_datasync_wm_name: datasync_wf
 workflow_parsing_wm_name: parsing_wf
 
-workflow_worker_kafka_url: 149.165.156.200:9092
+workflow_worker_kafka_url: "{{ kafka_url }}"
 workflow_worker_kafka_publisher_name: mft-status-publisher
 workflow_worker_kafka_publish_topic: mft-status-topic
 workflow_worker_kafka_consumner_group: mft-even-group

ansible/roles/kafka/defaults/main.yml

@@ -0,0 +1,26 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+
+kafka_package_name: "confluent-5.3.1"
+kafka_tgz_url: "http://packages.confluent.io/archive/5.3/confluent-5.3.1-2.12.zip"
+kafka_dir: "{{ datalake_deployment_dir }}/{{ kafka_package_name }}"
+kafka_rest_proxy_listener_port: 8082

ansible/roles/kafka/handlers/main.yml

@@ -0,0 +1,33 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+# Kafka related handlers
+- name: start kafka-rest-proxy
+  service: name=kafka-rest-proxy state=started enabled=yes
+  become: yes
+
+- name: stop kafka-rest-proxy
+  service: name=kafka-rest-proxy state=stopped enabled=yes
+  become: yes
+
+- name: restart kafka-rest-proxy
+  service: name=kafka-rest-proxy state=restarted enabled=yes
+  become: yes

ansible/roles/kafka/tasks/main.yml

@@ -0,0 +1,79 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+
+# Check for the availability of the Kafka package
+- name: Check kafka package availability
+  stat: path={{ kafka_dir }}
+  register: kafka_package
+  become: yes
+
+- name: Create deployment directory {{ datalake_deployment_dir }}
+  file: path={{datalake_deployment_dir}}
+    state=directory
+    mode=0755
+    owner={{ user }}
+    group={{ group }}
+  become: yes
+
+# Download Kafka
+- name: Download and unarchive Kafka from {{ kafka_tgz_url }}
+  unarchive: src="{{ kafka_tgz_url }}"
+    dest="{{ datalake_deployment_dir }}"
+    copy=no
+    owner="{{ user }}"
+    group="{{ group }}"
+  when: not kafka_package.stat.exists
+  become: yes
+
+# Create kafka logs directory
+- name: Create kafka logs directory
+  file: path="{{ kafka_dir }}/logs" state=directory owner={{ user }} group={{ group }}
+  become: yes
+
+# Config rest proxy and start
+- name: Copy kafka rest proxy properties file
+  template: src=kafka-rest.properties.j2
+    dest="{{ kafka_dir }}/etc/kafka-rest/kafka-rest.properties"
+    owner={{ user }}
+    group={{ group }}
+    mode="u=rw,g=r,o=r"
+  become: yes
+
+- name: open kafka rest proxy port
+  firewalld: port="{{ kafka_rest_proxy_listener_port }}/tcp"
+    zone=public permanent=true state=enabled immediate=yes
+  become: yes
+
+- name: systemd install kafka rest proxy service script
+  template: src=kafka-rest-proxy.service.j2
+    dest=/usr/lib/systemd/system/kafka-rest-proxy.service
+    owner={{ user }}
+    group={{ group }}
+    mode="u=rw,g=r,o=r"
+  become: yes
+
+- name: Reload systemd daemons
+  command: systemctl daemon-reload
+  notify: restart kafka-rest-proxy
+  become: yes
+
+...

ansible/roles/kafka/templates/kafka-rest-proxy.service.j2

@@ -0,0 +1,14 @@
+# {{ansible_managed}}
+
+[Unit]
+Description=Kafka-Rest-Proxy
+Before=
+After=network.target
+
+[Service]
+LOG_DIR={{ kafka_dir }}/logs
+ExecStart={{ kafka_dir }}/bin/kafka-rest-start {{ kafka_dir }}/etc/kafka-rest/kafka-rest.properties
+Restart=on-abort
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/kafka/templates/kafka-rest.properties.j2

@@ -0,0 +1,25 @@
+#
+# Copyright 2018 Confluent Inc.
+#
+# Licensed under the Confluent Community License (the "License"); you may not use
+# this file except in compliance with the License.  You may obtain a copy of the
+# License at
+#
+# http://www.confluent.io/confluent-community-license
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OF ANY KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations under the License.
+#
+
+#id=kafka-rest-test-server
+#schema.registry.url=http://localhost:8081
+#zookeeper.connect=localhost:2181
+bootstrap.servers=PLAINTEXT://{{ kafka_url }}
+listeners=http://{{ ansible_default_ipv4.address }}:{{ kafka_rest_proxy_listener_port }}
+#
+# Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center
+# Make sure that monitoring-interceptors-<version>.jar is on the Java class path
+#consumer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor
+#producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor