Adding the ansible inventory for production deployment

ansible/datalake.yml

@@ -26,6 +26,6 @@
     - kafka
     - custos
     - consul
-    - mft
     - data_lake
+    - mft
     - workflow-engine

ansible/inventories/prod-deployment/group_vars/all/vars.yml

@@ -0,0 +1,133 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+ansible_connection: ssh
+
+user: airavata
+group: airavata
+user_home: "/home/{{ user }}"
+
+# deployment related variables
+datalake_deployment_dir: "{{ user_home }}/data-lake-deployment"
+datalake_source_dir: "{{ user_home }}/data-lake-source"
+datalake_repo: "https://github.com/apache/airavata-data-lake.git"
+datalake_git_branch: master
+kafka_listener_port: 9092
+kafka_url: "{{ groups['data-lake'][0] }}:9092"
+custos_kafka_url: custos07.sciencegateways.iu.edu:9092
+zookeeper_client_port: 2181
+zookeeper_connection_url: localhost:2181
+
+mft_deployment_dir: "{{ user_home }}/mft-deployment"
+mft_source_dir: "{{ user_home }}/mft-source"
+mft_repo: "https://github.com/apache/airavata-mft.git"
+mft_git_branch: develop
+mft_distribution_version: 0.01
+
+custos_source_dir: "{{ user_home }}/custos-source"
+custos_repo: "https://github.com/apache/airavata-custos.git"
+custos_git_branch: develop
+
+mft_default_agent_id: agent0
+mft_default_agent_host: "{{ ansible_default_ipv4.address }}"
+mft_default_agent_advertised_url: https://beta.iubemcenter.scigap.org:8443/downloads
+mft_default_agent_port: 3333
+
+mft_api_service_host: localhost
+mft_api_service_http_port: 8088
+mft_api_service_grpc_port: 7004
+mft_api_service_id: 0
+
+mft_resource_service_host: localhost
+mft_resource_service_http_port: 8089
+mft_resource_service_grpc_port: 7002
+mft_resource_service_backend: org.apache.airavata.mft.resource.server.backend.datalake.DatalakeResourceBackend
+mft_resource_service_custos_client_id: "{{ vault_mft_resource_service_custos_client_id }}"
+mft_resource_service_custos_client_secret: "{{ vault_mft_resource_service_custos_client_secret }}"
+
+mft_secret_service_host: localhost
+mft_secret_service_http_port: 8081
+mft_secret_service_grpc_port: 7003
+mft_secret_service_backend: org.apache.airavata.mft.secret.server.backend.custos.CustosSecretBackend
+mft_secret_service_custos_client_id: "{{ vault_mft_resource_service_custos_client_id }}"
+mft_secret_service_custos_client_secret: "{{ vault_mft_resource_service_custos_client_secret }}"
+
+mft_consul_host: localhost
+mft_consul_port: 8500
+consul_version: 1.7.1
+consul_zip_file: consul_{{ consul_version }}_linux_amd64.zip
+consul_install_dir: /usr/local/bin
+consul_config_dir: /etc/consul
+consul_data_dir: /var/data
+
+datalake_distribution_version: 0.01-SNAPSHOT
+datalake_drms_host: localhost
+datalake_drms_grpc_port: 7070
+
+datalake_drms_neo4j_uri: bolt://scigap10.sciencegateways.iu.edu:7687
+datalake_drms_neo4j_user: "{{ vault_datalake_drms_neo4j_user }}"
+datalake_drms_neo4j_password: "{{ vault_datalake_drms_neo4j_password }}"
+datalake_drms_custos_client_id: "{{ vault_mft_resource_service_custos_client_id }}"
+datalake_drms_custos_client_secret: "{{ vault_mft_resource_service_custos_client_secret }}"
+datalake_drms_custos_host: services.usecustos.org
+datalake_drms_custos_port: 31161
+datalake_drms_http_port: 8900
+
+datalake_synch_broker_url: "{{ custos_kafka_url }}"
+datalake_synch_broker_consumer_group: custosEventsGroup
+datalake_synch_broker_topic: 10000201-24565f21-a87a-46ac-a50c-6cdf0e41028c
+datalake_synch_tenant_id: "{{ vault_workflow_manager_custos_id }}"
+
+datalake_data_orch_datasource_url: jdbc:mysql://localhost:3306/data_orchestrator?useSSL=false&serverTimezone=UTC&useLegacyDatetimeCode=false&createDatabaseIfNotExist=true&allowPublicKeyRetrieval=true
+datalake_data_orch_datasource_username: "{{ vault_datalake_data_orch_datasource_username }}"
+datalake_data_orch_datasource_password: "{{ vault_datalake_data_orch_datasource_password }}"
+datalake_data_orch_host: localhost
+datalake_data_orch_http_port: 8899
+datalake_data_orch_grpc_port: 6566
+datalake_data_orch_broker_url: "{{ kafka_url }}"
+datalake_data_orch_broker_consumer_group: DataOrchestratorGroup2
+datalake_data_orch_broker_topic: data-orchestrator-file-events
+datalake_data_orch_user_group: "{{ vault_datalake_data_orch_user_group }}"
+datalake_data_orch_admin_group: "{{ vault_datalake_data_orch_admin_group }}"
+
+workflow_deployment_dir: "{{ user_home }}/workflow-deployment"
+workflow_zookeeper_url: "{{ zookeeper_connection_url }}"
+workflow_helix_cluster_name: datalake
+
+workflow_datasync_wm_name: datasync_wf
+workflow_parsing_wm_name: parsing_wf
+
+workflow_worker_kafka_url: "{{ kafka_url }}"
+workflow_worker_kafka_publisher_name: mft-status-publisher
+workflow_worker_kafka_publish_topic: mft-status-topic
+workflow_worker_kafka_consumner_group: mft-even-group
+
+workflow_manager_host: localhost
+workflow_manager_grpc_port: 6565
+
+workflow_manager_mft_callback_url: "{{ groups['data-lake'][0] }}:33335"
+workflow_manager_mft_client_id: "{{ vault_workflow_manager_mft_client_id }}"
+workflow_manager_mft_client_secret: "{{ vault_workflow_manager_mft_client_secret }}"
+
+workflow_manager_custos_host: services.usecustos.org
+workflow_manager_custos_port: 31161
+workflow_manager_custos_id: "{{ vault_workflow_manager_custos_id }}"
+workflow_manager_custos_secret: "{{ vault_workflow_manager_custos_secret }}"

ansible/inventories/prod-deployment/group_vars/all/vault.yml

@@ -0,0 +1,160 @@
+$ANSIBLE_VAULT;1.1;AES256
+30333438336163376266336436643037656363626230383438343837373562306565326235376166
+3861613366306265326136663039323863626565613638340a633338313961626131376434363232
+36366437346164616530363461643435306536353765336664383732663034373036616666353437
+3163636433346461610a656436306162633266303333643035663331666339646264353030336662
+30356563386135383738343435346632313265306132333465386536316537373137376336636563
+63366637633035653136626437306263316165623039646462386436306465393039616362303233
+39346237633566316162363561636534373334373039303231643933666536333663303033333037
+37646162333036653164613936633939656562653233333533366431636261393566386539646232
+33353638313736373630326333326362366134323962353037373661633536336630366163313535
+61653032666264343138353861323364383464356230343564343233636162663064636430633833
+35346638363934363530643565663930616136613162393566626132613730383039373838373532
+39303235336339316637323964643764376265653366613964643135633638316665643862326335
+34623137363265613738636139643630356364336534646364633038333637623165343832366438
+37626339306339393066326265356265636135643631343637323830393333323533333036626264
+65393561323537646261303739373266353938643265663839343564373865663134643037343262
+38313366616166343338633064326462373765303936343166343465353161303464326363306166
+34306135396430656430343537363230336261323563643438363465343666653335373437316532
+65656533376139666264366365643838386639326564663562383865653163656265393737366666
+31383533656661653437326638623435366233663265643439313563616634663362333162356530
+64626261366232633831646635363563333035353038386535633931623833313461316237303833
+34633030616465643639323866326136333939623735346262313164313536626534393466333539
+64396432303432633237626631323039626661343233653030633063646432306665656135343763
+35303136303732643438663366646434386433386437613661383639326162653365623162616134
+36336131383033616362613832666366623130313037363233336236333836386163613264616366
+66376366633939643833353133316663653032316435363636623538323634376265393636663339
+66656636386535393338626435336366373165333237303530653030623738323564646338663431
+35616136646462353835396130356137613134326135333361633035616565656165386265653063
+31663435353633646462346337636466376537396130366463393531656530643239633937643636
+30666135646139393834613432373061393437373339306130363632616564616231643530373965
+36346161373530626137333562656237373234323066646538613439326334613339646238636336
+63633332353065613366623230623836333765633066303039363061343636353465613935623862
+37633238613866633133616562646366303061623738363136363231373637613337636265323365
+32663037373961626262313261633066613139663130666164626561613837373733333132313037
+66653430643761323830623463633535316137663433646132336631393133643236623338353666
+36356331366533396633623061346232643538346634626264363630633863326638343038636136
+32333239353962396266646532643065646636633164643934323732613730626565616137666632
+62313464323064613236333766376138656438643564626332633639316564626135613035616564
+36643835396266343063633139626262663733323638613930316265636430633435616238653237
+36626336356539373335393133656161366566396664363031323066653034333366336133313737
+66396266656331383164383266643430363939353832396361356262656530303263633830613131
+38663765376433633863386562363033653436306336323563376230653761393839663033333838
+32646234353533666635323862616430343034636638353966333332626233396661326334363930
+36396561313530373063353466343439333039336233383038303939656538353737333338643865
+64373039663165653236383838656335333238313930393931396238613134326535353563336264
+39653864323634393039646539366663616265653363356132646261623339656439383531636264
+38386636666633616266326130326336626139636535363735333031356430356533353738616238
+37663431663164376164366564643663393266353338663266343935613733393130666236363530
+38666561356463626261326662356566646162663732613366663036663232396163383532313266
+32396466383435373031366336313635373432333932396565663634333835653964653564313531
+62666237326665346630323733663835653634666561313833636335623862373832643930386337
+32613464306262303738393633323534366666313163356338343734353231323636653630363932
+34346234393337346565313735653566343363336166333162373863373830303761313465656232
+62623836303137636462373233303434386231373530623234383366323261386432303964316234
+62366230633539303735393139393665336431636462333834376532316263633839396535653239
+39336436343366643037386565353764303039313735623965333661323138336633303863666362
+38663830393035313066376530633565653137373564346166343765336336663035373632393736
+66386466353966643137613435376265356639353531313863316462393033376630613939353038
+36373539386261336134613134626563386232656634613034356362303034383061323765363337
+37666561613431633038336162376463326666346564643735306565363332393330643266663762
+34333630333561383632656332663965366330393063316665633937356562316339643833623366
+34623732653531656132303162376661656366616334636630346464646462626330306265393033
+62666665626539356431356362353563386438656237646139633663353038393838613738313532
+35643738306435356635613837663631366363323337613465656431336639346639316562356239
+35326164666530616635616265643635356434623365636465356138366565336436313733376539
+64303433386130633764356365353437646535356663646430346335353465636662376238623666
+33396239666339613230306666663035333564636261633365383830313463643064653734653030
+36396662373364613262626564373634666466356639623938333332646263323731353733373435
+31356463616261643064343535313631373334373632343536356637383735383665366436393163
+65306365306330383066636232336438363137613162613437663539313433653233346235303731
+62366134303935386335313864386563303663643930396364613861653937306663343633663033
+30373636633036663161353335633435643661623632356532343336313536363938656161613837
+32393564306433656239303563313039383862343134396337313738373039303832343933376133
+61376537363031356663613762303736363936646362383165636132393163663234346430383636
+38393564656166353337383266633837646139613232626537366165643634663532363639363238
+66353565393237346137636161623664646439396561336135366265623532633532633566386630
+38613234393566386263323338313834613664623262303037343330373665663162393265323932
+61316466616335333663303136383635393366666461666336663734636563633962373935613239
+33396633643933323564396430386237363461663862386634323962616462376531653532306561
+39646264613331653032383337653364643539343332313939636436303533663536656431306132
+33303364353339366264383338626130323036353538623538376236613163346430333536663638
+38343865643337396436353663383665303131323534326130373434323763666237663465656131
+61393638306531663764613165666435326533386364353133626661393535656364653234333536
+36383338653130326139613838303863363839383064313036383536356635316261356637396262
+37636237353332393433366664613335303339386665386466333631653631383232663032326234
+33646232633739353765396162323139636361323261396139376463373331616131343136613766
+36653732616239393633386562313036353735663263373862323761396532353634316433636265
+63306335313938306465316563303336666465303066336433383965393733343764366136663664
+61646631643835613232663934666233383439663037623931306335383465613265383031333035
+31313933363064643834363132636536313437336130386264343239386539353563353232336133
+65623639653230613061396536373661663134366261653465393438643861326237626139363563
+34343836353364666632663962616163373562333766396365663734613936313164633664653339
+62336633303465653231393461373331336634323166333830643132663930303230363333356436
+39316436313633663765383161623231383764636239363365623136373235653430653766616465
+37663338316162623961646133316235643638633539633230343037346665353062656437666237
+64636137383663653330366561656133623864363964376266633165656433373636396564663439
+37386162656333373935383362333930396134313939653837303861366433346536326565326562
+64616466306333306631383934303763363033303265633432383734353265313538376535653137
+33336130633836366365323035353034393661306133626631386465376362643233656364373235
+34306164373964633438383461656337373835346333626566623739353661313962326261636264
+32306236303233383831393939383435396334356363643537663831616238613961636438343464
+61363263383665633931363262303032626161653535613662343163333634393763323431366365
+39653538326331626538353365653130633630666232326639383430613265343137636561353238
+30653735386337356633386335393637383930343638636366313663383666343165383865666332
+65336661633636353461313230323438376565396639313561386336396539666363353863313339
+63633334333535323262383935383738393163656462316561623261373530323238313438613833
+65666563306132323866623364343464643166366133663337383037373938333064396535353863
+35616533333565316536393563613162346333303661643965313435353863616330373066633132
+64386330623861666435376234373066393064363265363361323331613037393062303935306237
+31386533383161343232353165616139323832376264626430623961653938653431383437643965
+65353761653264623731626165393661653838626434353261613335313639366530383734303962
+31653038363463316266326336306461393464633264323365643436336535626537363930613230
+36626637646264393566616362666266303234643938343535336262303261356336396533343665
+61653666636563363531353034333538373737353632386138626163356163656630623232366538
+61373361613532646335613462326461393437643534626537353335613636653363323565653034
+63316666616363636635343533356532303833643965636362646462313564356436653534323766
+33393432366232663664656563636163303530333439666437623735356636613966613430626538
+66326465346261393333323262633066346435623836356663623264303262346665343965336464
+38343930653633643638613833666166646136306266386338666132616131393338666562646439
+34616334383064393164313737623534333237613264313165343436343937653865393138396138
+31653333323631313237663132386564323139333731353931366137336466313237343664643861
+31353061343834613336366635386163336362613664316362303737663933386535633766386433
+63343135316634313363623765396138613562633437666233656563356533616665393635356264
+66323163336466343534326536666565306337666536353634663430653632646634646439656336
+63333835393162323234323565616261393861356264343634363563376638323333663166653832
+62346161636431646231323865643661303934363439346561373131343631333162613838376435
+38306637353163343464633530373139383436636338613831613738616435353930663530626638
+37336630353630343762373563626564663864306237613062666332633231303833333233303139
+62613935386432663634653238333230333236333866666233646637643934303636643333643337
+35313066363737616566333038396266396566366337393332646134386566316537366237336663
+31623134333761626238386432353565633935323361303662666431326164633562326635326534
+61383236623339346430336430613137636133363631383634393062366665343364316432316630
+62353539666362326333336531333930616132323237313630663732623731663531356230303837
+35313566633066383339646462326663333031393139666266356237303437363037363562633063
+34363265636233626339643733623731353737396339656466653965303963373032663966363262
+31383435306166343734336131356236643937383962373566313565373461333138326338623832
+61666635353663376631623561373061643565303133643466346261396336646266323866393532
+64336634316137646232313431343333633662613339383530316530373630656366666432306131
+65643263623435373334663239363232666362363863323863326437633839373063613439663062
+31326465646365626532636530663065663634633461333638376236306135376535313661383261
+30616161616662626365356339643731316236643966646533643233653137313538383533336365
+66393238376138616162343162366461636563366331356165653766626539343236313634646464
+30633961646231343361663836353363376565313034623866386230343638373066323631623761
+63373362323133323532623037356133636461663963323836306130303864636133616635386336
+31326131306362306532333334373537636238303431366234353737333362353732373532626130
+61376565613165646564666334313537383335316236396561646533623431633934353439396434
+64613533353436313736643738303230363031633032636466393934326333636133643531396436
+36343832366465363139636363653661636130323063623065626364346136313432353237636139
+33656366363861313665623066383062363532393864396331303434636439623635306131346233
+33386634353765346432376365646161393735373137343634613431653432376231623766346332
+31663065336461303835663639356338633931353030333763396266633065636163633064656637
+65633866303337663039386234643561656433326333383261383463646661363633303236646434
+65626437373836396466646633323332333564303563653466303331353262663061613734626331
+31386336306333343534666539346637323539346632396239366633323339666630646466353239
+33356433653530623836383030336138633439333264616664373665363538346564633862326639
+62633134376432616562646136633837383561633630313862393330346365353035336236396335
+30336662356232346466373366383137366332633736393764353862333537306638653663313835
+32333865393964303735636431383462386231653065613236613164626662366338653365666466
+33613365636433613933636430343731396536653934303933643134386531363761643961353431
+633235383864646166383832643466643139

ansible/inventories/prod-deployment/hosts

@@ -0,0 +1,2 @@
+[data-lake]
+scigap12.sciencegateways.iu.edu ansible_user=airavata