Skip to content
Permalink
Browse files
Merge branch 'master' of https://github.com/apache/airavata-data-lake
…into sharing_service_impl
  • Loading branch information
isururanawaka committed Sep 8, 2021
2 parents 8a7581f + 83ce467 commit ba5acab7027f0e817b7d9afdd8e49d9b756b815e
Showing 25 changed files with 694 additions and 179 deletions.
@@ -22,9 +22,10 @@
- hosts: data-lake
roles:
- env_setup
- zookeeper
- kafka
# - custos
# - consul
# - mft
# - data_lake
# - workflow-engine
- custos
- consul
- mft
- data_lake
- workflow-engine
@@ -30,7 +30,10 @@ datalake_deployment_dir: "{{ user_home }}/data-lake-deployment"
datalake_source_dir: "{{ user_home }}/data-lake-source"
datalake_repo: "https://github.com/apache/airavata-data-lake.git"
datalake_git_branch: master
kafka_url: 149.165.157.235:9092
kafka_listener_port: 9092
kafka_url: "{{ groups['data-lake'][0] }}:9092"
zookeeper_client_port: 2181
zookeeper_connection_url: localhost:2181

mft_deployment_dir: "{{ user_home }}/mft-deployment"
mft_source_dir: "{{ user_home }}/mft-source"
@@ -43,7 +46,7 @@ custos_repo: "https://github.com/apache/airavata-custos.git"
custos_git_branch: develop

mft_default_agent_id: agent0
mft_default_agent_host: 10.1.0.42
mft_default_agent_host: "{{ ansible_default_ipv4.address }}"
mft_default_agent_advertised_url: https://beta.iubemcenter.scigap.org:8443/downloads
mft_default_agent_port: 3333

@@ -62,7 +65,9 @@ mft_resource_service_custos_client_secret: "{{ vault_mft_resource_service_custos
mft_secret_service_host: localhost
mft_secret_service_http_port: 8081
mft_secret_service_grpc_port: 7003
mft_secret_service_backend: org.apache.airavata.mft.secret.server.backend.file.FileBasedSecretBackend
mft_secret_service_backend: org.apache.airavata.mft.secret.server.backend.custos.CustosSecretBackend
mft_secret_service_custos_client_id: "{{ vault_mft_resource_service_custos_client_id }}"
mft_secret_service_custos_client_secret: "{{ vault_mft_resource_service_custos_client_secret }}"

mft_consul_host: localhost
mft_consul_port: 8500
@@ -103,7 +108,7 @@ datalake_data_orch_user_group: emc-users_e8a37f6d-e3b3-4e4a-9081-265bb42b1b99
datalake_data_orch_admin_group: admin-group_c1aac070-1512-4e98-b1d1-e06a94f03665

workflow_deployment_dir: "{{ user_home }}/workflow-deployment"
workflow_zookeeper_url: localhost:2181
workflow_zookeeper_url: "{{ zookeeper_connection_url }}"
workflow_helix_cluster_name: datalake

workflow_datasync_wm_name: datasync_wf
@@ -117,7 +122,7 @@ workflow_worker_kafka_consumner_group: mft-even-group
workflow_manager_host: localhost
workflow_manager_grpc_port: 6565

workflow_manager_mft_callback_url: 149.165.157.235:33335
workflow_manager_mft_callback_url: "{{ groups['data-lake'][0] }}:33335"
workflow_manager_mft_client_id: "{{ vault_workflow_manager_mft_client_id }}"
workflow_manager_mft_client_secret: "{{ vault_workflow_manager_mft_client_secret }}"

Large diffs are not rendered by default.

@@ -1,2 +1,2 @@
[data-lake]
149.165.157.235 ansible_user=ubuntu
149.165.156.22 ansible_user=ubuntu
@@ -33,6 +33,11 @@
zone=public permanent=true state=enabled immediate=yes
become: yes

- name: open firewall port {{ datalake_data_orch_grpc_port }} for Data Orchestrator gRPC connections
firewalld: port="{{ datalake_data_orch_grpc_port }}/tcp"
zone=public permanent=true state=enabled immediate=yes
become: yes

- name: Create Datalake deployment directory {{ datalake_deployment_dir }}
become: yes
file: path={{ datalake_deployment_dir }}
@@ -77,6 +82,18 @@
become: yes
become_user: "{{ user }}"

- name: Delete lib directories of the distributions
file:
state: absent
path: "{{ datalake_deployment_dir }}/{{ item }}"
with_items:
- "DRMS-{{datalake_distribution_version}}/lib"
- "DRMS-Custos-Synchronizer-{{datalake_distribution_version}}/lib"
- "Orchestrator-API-Service-{{datalake_distribution_version}}/lib"
become: yes
become_user: "{{ user }}"
ignore_errors: yes

- name: Copy Datalake distributions to Datalake deployment directory
unarchive: "src={{ datalake_source_dir }}/{{ item }}
dest={{ datalake_deployment_dir }}/ copy=no"
@@ -23,4 +23,21 @@
kafka_package_name: "confluent-5.3.1"
kafka_tgz_url: "http://packages.confluent.io/archive/5.3/confluent-5.3.1-2.12.zip"
kafka_dir: "{{ datalake_deployment_dir }}/{{ kafka_package_name }}"
kafka_rest_proxy_listener_port: 8082
kafka_rest_proxy_listener_port: 8082

broker_id: "0"
network_threads_count: "3"
io_threads_count: "8"
socket_send_buf_bytes: "102400"
socket_receive_buf_bytes: "102400"
socket_request_max_bytes: "104857600"
num_partitions: "1"
num_recovery_threads: "1"
offsets_topic_replication_fac: "1"
trans_state_log_replication_fac: "1"
trans_state_log: "1"
log_retention_hrs: "168"
log_segment_bytes: "1073741824"
log_retention_check_interval: "300000"
grp_initial_rebalance_delay: "0"
kafka_listener_port: 9092
@@ -20,6 +20,18 @@

---
# Kafka related handlers
- name: start kafka
service: name=kafka state=started enabled=yes
become: yes

- name: stop kafka
service: name=kafka state=stopped enabled=yes
become: yes

- name: restart kafka
service: name=kafka state=restarted enabled=yes
become: yes

- name: start kafka-rest-proxy
service: name=kafka-rest-proxy state=started enabled=yes
become: yes
@@ -49,6 +49,17 @@
file: path="{{ kafka_dir }}/logs" state=directory owner={{ user }} group={{ group }}
become: yes


# Config kafka server and start
- name: Copy kafka server properties file
template: src=server.properties.j2
dest="{{ kafka_dir }}/etc/kafka/server.properties"
owner={{ user }}
group={{ group }}
mode="u=rw,g=r,o=r"
notify: restart kafka
become: yes

# Config rest proxy and start
- name: Copy kafka rest proxy properties file
template: src=kafka-rest.properties.j2
@@ -58,11 +69,25 @@
mode="u=rw,g=r,o=r"
become: yes

- name: open kafka proxy port
firewalld: port="{{ kafka_listener_port }}/tcp"
zone=public permanent=true state=enabled immediate=yes
become: yes

- name: open kafka rest proxy port
firewalld: port="{{ kafka_rest_proxy_listener_port }}/tcp"
zone=public permanent=true state=enabled immediate=yes
become: yes

- name: systemd install kafka service script
template: src=kafka.service.j2
dest=/usr/lib/systemd/system/kafka.service
owner={{ user }}
group={{ group }}
mode="u=rw,g=r,o=r"
notify: start kafka
become: yes

- name: systemd install kafka rest proxy service script
template: src=kafka-rest-proxy.service.j2
dest=/usr/lib/systemd/system/kafka-rest-proxy.service
@@ -71,6 +96,11 @@
mode="u=rw,g=r,o=r"
become: yes

- name: Reload systemd daemons
command: systemctl daemon-reload
notify: restart kafka
become: yes

- name: Reload systemd daemons
command: systemctl daemon-reload
notify: restart kafka-rest-proxy
@@ -0,0 +1,14 @@
# {{ansible_managed}}

[Unit]
Description=Kafka
Before=
After=network.target

[Service]
LOG_DIR={{ kafka_dir }}/logs
ExecStart={{ kafka_dir }}/bin/kafka-server-start {{ kafka_dir }}/etc/kafka/server.properties
Restart=on-abort

[Install]
WantedBy=multi-user.target

0 comments on commit ba5acab

Please sign in to comment.