Update DA cluster tutorial

tutorials/dataavenue-cluster/infra-dataavenue.yaml

@@ -9,10 +9,9 @@ nodes:
     name: storage
     type: storage_node
     scaling:
-      min: 2
-      max: 10
+      min: 1
+      max: 1
     variables:
-      storage_user_name: testuser
       access_key: A8Q2WPCWAELW61RWDGO8
       secret_key: FWd1mccBfnw6VHa2vod98NEQktRCYlCronxbO1aQ
 

tutorials/dataavenue-cluster/nodes/cloud_init_dataavenue.yaml

@@ -1,59 +1,25 @@
 #cloud-config
 write_files:
 ################################
-# SCRIPT TO INSTALL DOCKER
-################################
-- path: /bin/deploy-docker.sh
-  content: |
-    #!/bin/bash
-    echo "Install DOCKER starts."
-    set -x
-    apt-get update
-    apt-get install -y --no-install-recommends linux-image-extra-$(uname -r) linux-image-extra-virtual apt-transport-https ca-certificates curl software-properties-common
-    echo deb http://apt.dockerproject.org/repo ubuntu-trusty main > /etc/apt/sources.list.d/docker.list
-    curl -fsSL https://apt.dockerproject.org/gpg | apt-key add -
-    add-apt-repository "deb https://apt.dockerproject.org/repo/ ubuntu-$(lsb_release -cs) main"
-    apt-get update
-    apt-get install -y docker-engine
-    echo "DOCKER_OPTS='-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock'" > /etc/default/docker
-    service docker restart
-    echo "Install DOCKER stops."
-  permissions: '755'
-
-#####################################
-# SCRIPT TO INSTALL DOCKER-COMPOSE
-#####################################
-- path: /bin/deploy-docker-compose.sh
-  content: |
-    #!/bin/bash
-    set -x
-    echo "Install DOCKER-COMPOSE starts."
-    sudo curl -L https://github.com/docker/compose/releases/download/1.16.1/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
-    sudo chmod +x /usr/local/bin/docker-compose
-    echo "Install DOCKER-COMPOSE stops."
-  permissions: '755'
-
-################################
-# SCRIPT TO INSTALL DATAAVENUE
+# SCRIPT TO INSTALL AND DEPLOY DATAAVENUE
 ################################
 - path: /bin/deploy-dataavenue.sh
   content: |
     #!/bin/bash
-    set -x
-    echo "Install DATAAVENUE starts."
-    wget -O data-avenue-docker-compose-latest.tar.gz https://nextcloud.sztaki.hu/s/EiNnjwDjR9xfTQZ/download --directory /home/ubuntu/
-    cd /home/ubuntu
-    tar zxvf data-avenue-docker-compose-latest.tar.gz
-    cd /home/ubuntu/data-avenue-docker-compose
-    docker-compose up > dataavenue.out &
-    echo "Install DATAAVENUE stops."
+    echo "Deploying DATAAVENUE..."
+    apt-get update
+    git clone https://github.com/SZTAKI-LPDS/data-avenue.git /home/ubuntu/data-avenue
+    mvn -f /home/ubuntu/data-avenue/pom.xml package
+    cp /home/ubuntu/data-avenue/data-avenue.core.war/target/dataavenue.war /home/ubuntu/data-avenue/data-avenue.docker-compose/dataavenue/webapps/
+    docker-compose -f /home/ubuntu/data-avenue/data-avenue.docker-compose/docker-compose.yml up -d
+    echo "Done."
   permissions: '755'
-
+packages:
+ - git
+ - maven
+ - openjdk-8-jdk
+ - docker-compose
 runcmd:
-#Install DOCKER
-- /bin/deploy-docker.sh
-#Install DOCKER-COMPOSE
-- /bin/deploy-docker-compose.sh
 #Run Dataavenue
 - /bin/deploy-dataavenue.sh
-- echo "DATAAAVENUE NODES'S CONTEXTUALIZATION DONE."
+- echo "DATAAVENUE NODE CONTEXTUALIZATION DONE."

tutorials/dataavenue-cluster/nodes/cloud_init_storage.yaml

@@ -1,34 +1,19 @@
 #cloud-config
 write_files:
 ################################
-# SCRIPT TO INSTALL DOCKER
+# SCRIPT TO INSTALL AND DEPLOY MINIO
 ################################
-- path: /bin/deploy-docker.sh
+- path: /bin/deploy-minio.sh
   content: |
     #!/bin/bash
-    echo "Install DOCKER starts."
-    set -x
+    echo "Deploying MINIO..."
     apt-get update
-    apt-get install -y --no-install-recommends linux-image-extra-$(uname -r) linux-image-extra-virtual apt-transport-https ca-certificates curl software-properties-common
-    echo deb http://apt.dockerproject.org/repo ubuntu-trusty main > /etc/apt/sources.list.d/docker.list
-    curl -fsSL https://apt.dockerproject.org/gpg | apt-key add -
-    add-apt-repository "deb https://apt.dockerproject.org/repo/ ubuntu-$(lsb_release -cs) main"
-    apt-get update
-    apt-get install -y docker-engine
-    echo "DOCKER_OPTS='-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock'" > /etc/default/docker
-    service docker restart
-    echo "Install DOCKER stops."
+    docker run -d -v /mnt:/data -p 80:9000/tcp --name minio -e "MINIO_ACCESS_KEY={{variables.access_key}}" -e "MINIO_SECRET_KEY={{variables.secret_key}}" minio/minio server /data
+    echo "Done."
   permissions: '755'
-
+packages:
+ - docker-compose
 runcmd:
-#Install DOCKER
-- /bin/deploy-docker.sh
-#Install CEPH NETWORK
-- docker network inspect ceph 2>&1 > /dev/null || docker network create --driver bridge --subnet 172.16.13.0/28 --gateway 172.16.13.1 ceph
-- mkdir -p /etc/ceph
-#Run CEPH CONTAINER
-- docker run -d --name=ceph --net=ceph -e CEPH_DEMO_UID={{variables.storage_user_name}} -e CEPH_DEMO_ACCESS_KEY={{variables.access_key}} -e  CEPH_DEMO_SECRET_KEY={{variables.secret_key}} -e MON_IP=172.16.13.2 -e CEPH_NETWORK=172.16.13.0/28 -e CEPH_PUBLIC_NETWORK=172.16.13.0/28 -p 80:80 -p 5000:5000 -p 6789:6789 -p 6800-6805:6800-6805 -v /etc/ceph:/etc/ceph  ceph/demo:tag-build-master-jewel-ubuntu-16.04
-#- docker exec ceph ceph -s 
-#Create user to the S3 storage
-#- docker exec ceph radosgw-admin user create --uid={{variables.storage_user_name}} --display-name="{{variables.storage_user_name}}" --access-key={{variables.access_key}} --secret={{variables.secret_key}} > .env
-- echo "STORAGE NODE'S CONTEXTUALISATION DONE"
+#Run MINIO
+- /bin/deploy-minio.sh
+- echo "STORAGE NODE CONTEXTUALISATION DONE"

tutorials/dataavenue-cluster/nodes/node_definitions.yaml

@@ -2,38 +2,31 @@
   -
     resource:
       type: nova
-      endpoint: replace_with_endpoint_of_nova_interface_of_your_cloud
-      project_id: replace_with_projectid_to_use
+      endpoint: https://sztaki.cloud.mta.hu:5000/v3
+      project_id: a9c30db63ddf47a98045ef9c726c7436
       user_domain_name: Default
-      image_id: replace_with_id_of_your_image_on_your_target_cloud
-      network_id: replace_with_id_of_network_on_your_target_cloud
-      flavor_name: replace_with_id_of_the_flavor_on_your_target_cloud
-      key_name: replace_with_name_of_keypair_or_remove
-      security_groups:
-        -
-          replace_with_security_group_to_add_or_remove_section
-      floating_ip: add_yes_if_you_need_floating_ip_or_remove
-      floating_ip_pool: replace_with_name_of_floating_ip_pool_or_remove
+      image_id: 6bba6dc3-b6d5-4b15-942e-61e0ef2f93cb
+      network_id: 01efee1c-858c-4047-a48a-e2fab056f82a
+      flavor_name: 3
+      key_name: attila-key
+      security_groups: [Open]
     contextualisation:
       type: cloudinit
       context_template: !yaml_import
         url: file://cloud_init_dataavenue.yaml
+
 'node_def:storage_node':
   -
     resource:
       type: nova
-      endpoint: replace_with_endpoint_of_nova_interface_of_your_cloud
-      project_id: replace_with_projectid_to_use
+      endpoint: https://sztaki.cloud.mta.hu:5000/v3
+      project_id: a9c30db63ddf47a98045ef9c726c7436
       user_domain_name: Default
-      image_id: replace_with_id_of_your_image_on_your_target_cloud
-      network_id: replace_with_id_of_network_on_your_target_cloud
-      flavor_name: replace_with_id_of_the_flavor_on_your_target_cloud
-      key_name: replace_with_name_of_keypair_or_remove
-      security_groups:
-        -
-          replace_with_security_group_to_add_or_remove_section
-      floating_ip: add_yes_if_you_need_floating_ip_or_remove
-      floating_ip_pool: replace_with_name_of_floating_ip_pool_or_remove
+      image_id: 6bba6dc3-b6d5-4b15-942e-61e0ef2f93cb
+      network_id: 01efee1c-858c-4047-a48a-e2fab056f82a
+      flavor_name: 3
+      key_name: attila-key
+      security_groups: [Open]
     contextualisation:
       type: cloudinit
       context_template: !yaml_import