Skip to content
Permalink
Browse files
Generalizing ansible vars, adding zookeepr role, kafka installation, …
…migrating to new backup instance
  • Loading branch information
DImuthuUpe committed Sep 7, 2021
1 parent 87ead7f commit e76ee502d6c4f85629b856a2f33067c83b92b397
Showing 18 changed files with 653 additions and 171 deletions.
@@ -30,7 +30,10 @@ datalake_deployment_dir: "{{ user_home }}/data-lake-deployment"
datalake_source_dir: "{{ user_home }}/data-lake-source"
datalake_repo: "https://github.com/apache/airavata-data-lake.git"
datalake_git_branch: master
kafka_url: 149.165.157.235:9092
kafka_listener_port: 9092
kafka_url: "{{ groups['data-lake'][0] }}:9092"
zookeeper_client_port: 2181
zookeeper_connection_url: localhost:2181

mft_deployment_dir: "{{ user_home }}/mft-deployment"
mft_source_dir: "{{ user_home }}/mft-source"
@@ -43,7 +46,7 @@ custos_repo: "https://github.com/apache/airavata-custos.git"
custos_git_branch: develop

mft_default_agent_id: agent0
mft_default_agent_host: 10.1.0.42
mft_default_agent_host: "{{ ansible_default_ipv4.address }}"
mft_default_agent_advertised_url: https://beta.iubemcenter.scigap.org:8443/downloads
mft_default_agent_port: 3333

@@ -62,7 +65,9 @@ mft_resource_service_custos_client_secret: "{{ vault_mft_resource_service_custos
mft_secret_service_host: localhost
mft_secret_service_http_port: 8081
mft_secret_service_grpc_port: 7003
mft_secret_service_backend: org.apache.airavata.mft.secret.server.backend.file.FileBasedSecretBackend
mft_secret_service_backend: org.apache.airavata.mft.secret.server.backend.custos.CustosSecretBackend
mft_secret_service_custos_client_id: "{{ vault_mft_resource_service_custos_client_id }}"
mft_secret_service_custos_client_secret: "{{ vault_mft_resource_service_custos_client_secret }}"

mft_consul_host: localhost
mft_consul_port: 8500
@@ -103,7 +108,7 @@ datalake_data_orch_user_group: emc-users_e8a37f6d-e3b3-4e4a-9081-265bb42b1b99
datalake_data_orch_admin_group: admin-group_c1aac070-1512-4e98-b1d1-e06a94f03665

workflow_deployment_dir: "{{ user_home }}/workflow-deployment"
workflow_zookeeper_url: localhost:2181
workflow_zookeeper_url: "{{ zookeeper_connection_url }}"
workflow_helix_cluster_name: datalake

workflow_datasync_wm_name: datasync_wf
@@ -117,7 +122,7 @@ workflow_worker_kafka_consumner_group: mft-even-group
workflow_manager_host: localhost
workflow_manager_grpc_port: 6565

workflow_manager_mft_callback_url: 149.165.157.235:33335
workflow_manager_mft_callback_url: "{{ groups['data-lake'][0] }}:33335"
workflow_manager_mft_client_id: "{{ vault_workflow_manager_mft_client_id }}"
workflow_manager_mft_client_secret: "{{ vault_workflow_manager_mft_client_secret }}"

Large diffs are not rendered by default.

@@ -1,2 +1,2 @@
[data-lake]
149.165.157.235 ansible_user=ubuntu
149.165.156.22 ansible_user=ubuntu
@@ -33,6 +33,11 @@
zone=public permanent=true state=enabled immediate=yes
become: yes

- name: open firewall port {{ datalake_data_orch_grpc_port }} for Data Orchestrator gRPC connections
firewalld: port="{{ datalake_data_orch_grpc_port }}/tcp"
zone=public permanent=true state=enabled immediate=yes
become: yes

- name: Create Datalake deployment directory {{ datalake_deployment_dir }}
become: yes
file: path={{ datalake_deployment_dir }}
@@ -77,6 +82,18 @@
become: yes
become_user: "{{ user }}"

- name: Delete lib directories of the distributions
file:
state: absent
path: "{{ datalake_deployment_dir }}/{{ item }}"
with_items:
- "DRMS-{{datalake_distribution_version}}/lib"
- "DRMS-Custos-Synchronizer-{{datalake_distribution_version}}/lib"
- "Orchestrator-API-Service-{{datalake_distribution_version}}/lib"
become: yes
become_user: "{{ user }}"
ignore_errors: yes

- name: Copy Datalake distributions to Datalake deployment directory
unarchive: "src={{ datalake_source_dir }}/{{ item }}
dest={{ datalake_deployment_dir }}/ copy=no"
@@ -23,4 +23,21 @@
kafka_package_name: "confluent-5.3.1"
kafka_tgz_url: "http://packages.confluent.io/archive/5.3/confluent-5.3.1-2.12.zip"
kafka_dir: "{{ datalake_deployment_dir }}/{{ kafka_package_name }}"
kafka_rest_proxy_listener_port: 8082
kafka_rest_proxy_listener_port: 8082

broker_id: "0"
network_threads_count: "3"
io_threads_count: "8"
socket_send_buf_bytes: "102400"
socket_receive_buf_bytes: "102400"
socket_request_max_bytes: "104857600"
num_partitions: "1"
num_recovery_threads: "1"
offsets_topic_replication_fac: "1"
trans_state_log_replication_fac: "1"
trans_state_log: "1"
log_retention_hrs: "168"
log_segment_bytes: "1073741824"
log_retention_check_interval: "300000"
grp_initial_rebalance_delay: "0"
kafka_listener_port: 9092
@@ -20,6 +20,18 @@

---
# Kafka related handlers
- name: start kafka
service: name=kafka state=started enabled=yes
become: yes

- name: stop kafka
service: name=kafka state=stopped enabled=yes
become: yes

- name: restart kafka
service: name=kafka state=restarted enabled=yes
become: yes

- name: start kafka-rest-proxy
service: name=kafka-rest-proxy state=started enabled=yes
become: yes
@@ -49,6 +49,17 @@
file: path="{{ kafka_dir }}/logs" state=directory owner={{ user }} group={{ group }}
become: yes


# Config kafka server and start
- name: Copy kafka server properties file
template: src=server.properties.j2
dest="{{ kafka_dir }}/etc/kafka/server.properties"
owner={{ user }}
group={{ group }}
mode="u=rw,g=r,o=r"
notify: restart kafka
become: yes

# Config rest proxy and start
- name: Copy kafka rest proxy properties file
template: src=kafka-rest.properties.j2
@@ -58,11 +69,25 @@
mode="u=rw,g=r,o=r"
become: yes

- name: open kafka proxy port
firewalld: port="{{ kafka_listener_port }}/tcp"
zone=public permanent=true state=enabled immediate=yes
become: yes

- name: open kafka rest proxy port
firewalld: port="{{ kafka_rest_proxy_listener_port }}/tcp"
zone=public permanent=true state=enabled immediate=yes
become: yes

- name: systemd install kafka service script
template: src=kafka.service.j2
dest=/usr/lib/systemd/system/kafka.service
owner={{ user }}
group={{ group }}
mode="u=rw,g=r,o=r"
notify: start kafka
become: yes

- name: systemd install kafka rest proxy service script
template: src=kafka-rest-proxy.service.j2
dest=/usr/lib/systemd/system/kafka-rest-proxy.service
@@ -71,6 +96,11 @@
mode="u=rw,g=r,o=r"
become: yes

- name: Reload systemd daemons
command: systemctl daemon-reload
notify: restart kafka
become: yes

- name: Reload systemd daemons
command: systemctl daemon-reload
notify: restart kafka-rest-proxy
@@ -0,0 +1,14 @@
# {{ansible_managed}}

[Unit]
Description=Kafka
Before=
After=network.target

[Service]
LOG_DIR={{ kafka_dir }}/logs
ExecStart={{ kafka_dir }}/bin/kafka-server-start {{ kafka_dir }}/etc/kafka/server.properties
Restart=on-abort

[Install]
WantedBy=multi-user.target
@@ -0,0 +1,195 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# see kafka.server.KafkaConfig for additional details and defaults

############################# Server Basics #############################

# The id of the broker. This must be set to a unique integer for each broker.
broker.id={{ broker_id }}

############################# Socket Server Settings #############################

# The address the socket server listens on. It will get the value returned from
# java.net.InetAddress.getCanonicalHostName() if not configured.
# FORMAT:
# listeners = listener_name://host_name:port
# EXAMPLE:
# listeners = PLAINTEXT://your.host.name:9092
listeners=PLAINTEXT://{{ ansible_default_ipv4.address }}:{{ kafka_listener_port }}

# Hostname and port the broker will advertise to producers and consumers. If not set,
# it uses the value for "listeners" if configured. Otherwise, it will use the value
# returned from java.net.InetAddress.getCanonicalHostName().
#advertised.listeners=PLAINTEXT://your.host.name:9092

# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL

# The number of threads that the server uses for receiving requests from the network and sending responses to the network
num.network.threads={{ network_threads_count }}

# The number of threads that the server uses for processing requests, which may include disk I/O
num.io.threads={{ io_threads_count }}

# The send buffer (SO_SNDBUF) used by the socket server
socket.send.buffer.bytes={{ socket_send_buf_bytes }}

# The receive buffer (SO_RCVBUF) used by the socket server
socket.receive.buffer.bytes={{ socket_receive_buf_bytes }}

# The maximum size of a request that the socket server will accept (protection against OOM)
socket.request.max.bytes={{ socket_request_max_bytes }}


############################# Log Basics #############################

# A comma separated list of directories under which to store log files
log.dirs={{ kafka_dir }}/logs

# The default number of log partitions per topic. More partitions allow greater
# parallelism for consumption, but this will also result in more files across
# the brokers.
num.partitions={{ num_partitions }}

# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
# This value is recommended to be increased for installations with data dirs located in RAID array.
num.recovery.threads.per.data.dir={{ num_recovery_threads }}

############################# Internal Topic Settings #############################
# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
# For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3.
offsets.topic.replication.factor={{ offsets_topic_replication_fac }}
transaction.state.log.replication.factor={{ trans_state_log_replication_fac }}
transaction.state.log.min.isr={{ trans_state_log }}

############################# Log Flush Policy #############################

# Messages are immediately written to the filesystem but by default we only fsync() to sync
# the OS cache lazily. The following configurations control the flush of data to disk.
# There are a few important trade-offs here:
# 1. Durability: Unflushed data may be lost if you are not using replication.
# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks.
# The settings below allow one to configure the flush policy to flush data after a period of time or
# every N messages (or both). This can be done globally and overridden on a per-topic basis.

# The number of messages to accept before forcing a flush of data to disk
#log.flush.interval.messages=10000

# The maximum amount of time a message can sit in a log before we force a flush
#log.flush.interval.ms=1000

############################# Log Retention Policy #############################

# The following configurations control the disposal of log segments. The policy can
# be set to delete segments after a period of time, or after a given size has accumulated.
# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
# from the end of the log.

# The minimum age of a log file to be eligible for deletion due to age
log.retention.hours={{ log_retention_hrs }}

# A size-based retention policy for logs. Segments are pruned from the log unless the remaining
# segments drop below log.retention.bytes. Functions independently of log.retention.hours.
#log.retention.bytes=1073741824

# The maximum size of a log segment file. When this size is reached a new log segment will be created.
log.segment.bytes={{ log_segment_bytes }}

# The interval at which log segments are checked to see if they can be deleted according
# to the retention policies
log.retention.check.interval.ms={{ log_retention_check_interval }}

############################# Zookeeper #############################

# Zookeeper connection string (see zookeeper docs for details).
# This is a comma separated host:port pairs, each corresponding to a zk
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
# You can also append an optional chroot string to the urls to specify the
# root directory for all kafka znodes.
zookeeper.connect={{ zookeeper_connection_url }}

# Timeout in ms for connecting to zookeeper
zookeeper.connection.timeout.ms=6000

##################### Confluent Metrics Reporter #######################
# Confluent Control Center and Confluent Auto Data Balancer integration
#
# Uncomment the following lines to publish monitoring data for
# Confluent Control Center and Confluent Auto Data Balancer
# If you are using a dedicated metrics cluster, also adjust the settings
# to point to your metrics kakfa cluster.
#metric.reporters=io.confluent.metrics.reporter.ConfluentMetricsReporter
#confluent.metrics.reporter.bootstrap.servers=localhost:9092
#
# Uncomment the following line if the metrics cluster has a single broker
#confluent.metrics.reporter.topic.replicas=1

##################### Confluent Proactive Support ######################
# If set to true, and confluent-support-metrics package is installed
# then the feature to collect and report support metrics
# ("Metrics") is enabled. If set to false, the feature is disabled.
#
confluent.support.metrics.enable=true


# The customer ID under which support metrics will be collected and
# reported.
#
# When the customer ID is set to "anonymous" (the default), then only a
# reduced set of metrics is being collected and reported.
#
# Confluent customers
# -------------------
# If you are a Confluent customer, then you should replace the default
# value with your actual Confluent customer ID. Doing so will ensure
# that additional support metrics will be collected and reported.
#
confluent.support.customer.id=anonymous

############################# Group Coordinator Settings #############################

# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance.
# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms.
# The default value for this is 3 seconds.
# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
group.initial.rebalance.delay.ms={{ grp_initial_rebalance_delay }}


############################# Confluent Authorizer Settings #############################

# Uncomment to enable Confluent Authorizer with support for ACLs, LDAP groups and RBAC
#authorizer.class.name=io.confluent.kafka.security.authorizer.ConfluentServerAuthorizer
# Semi-colon separated list of super users in the format <principalType>:<principalName>
#super.users=
# Specify a valid Confluent license. By default free-tier license will be used
#confluent.license=
# Replication factor for the topic used for licensing. Default is 3.
confluent.license.topic.replication.factor=1

# Uncomment the following lines and specify values where required to enable RBAC
# Enable RBAC provider
#confluent.authorizer.access.rule.providers=ACL,RBAC
# Bootstrap servers for RBAC metadata. Must be provided if this broker is not in the metadata cluster
#confluent.metadata.bootstrap.servers=PLAINTEXT://127.0.0.1:9092
# Replication factor for the metadata topic used for authorization. Default is 3.
confluent.metadata.topic.replication.factor=1

# Listeners for metadata server
#confluent.metadata.server.listeners=http://0.0.0.0:8090
# Advertised listeners for metadata server
#confluent.metadata.server.advertised.listeners=http://127.0.0.1:8090

0 comments on commit e76ee50

Please sign in to comment.