Merge branch 'devel'

occopus · Mar 1, 2021 · ca0d864 · ca0d864
2 parents 7c3f439 + 79e25d2
commit ca0d864
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 63 deletions.
diff --git a/tutorials/hadoop-cluster.tar.gz b/tutorials/hadoop-cluster.tar.gz
diff --git a/tutorials/hadoop-cluster/infra-hadoop-cluster.yaml b/tutorials/hadoop-cluster/infra-hadoop-cluster.yaml
@@ -13,10 +13,11 @@ nodes:
       max: 10
 
 variables:
-  HADOOP_VERSION: 2.10.1
+  HADOOP_VERSION: 3.3.0
   CONSUL_VERSION: 1.9.1
   CONSUL_TEMPLATE_VERSION: 0.25.1
 
 dependencies:
   -
     connection: [ *S, *M ]
+
diff --git a/tutorials/hadoop-cluster/nodes/cloud_init_hadoop_master.yaml b/tutorials/hadoop-cluster/nodes/cloud_init_hadoop_master.yaml
@@ -35,7 +35,7 @@ write_files:
 
     export DEBIAN_FRONTEND=noninteractive
     apt-get update
-    apt-get install -y openjdk-8-jdk openjdk-8-jre python3-pip unzip
+    apt-get install -yq --no-install-recommends openjdk-8-jdk openjdk-8-jre unzip
     echo "Install requirement packages starts."
 
 
@@ -76,7 +76,6 @@ write_files:
     chown hduser:hduser /home/hduser/.bashrc
     echo export PATH="/home/hduser/hadoop/bin:$PATH" >> /home/hduser/.bashrc
     mv /tmp/hadoop/configs/* /home/hduser/hadoop/etc/hadoop
-    mv /tmp/hadoop/webconfigs/* /home/hduser/hadoop/share/hadoop/hdfs/webapps/hdfs/WEB-INF/
     echo "hadoop: lpds, admin" >> /home/hduser/hadoop/etc/hadoop/realm.properties
     echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre" >> /home/hduser/hadoop/etc/hadoop/hadoop-env.sh
     echo "export HADOOP_PID_DIR=/home/hduser/hadoop" >> /home/hduser/hadoop/etc/hadoop/hadoop-env.sh
@@ -124,12 +123,11 @@ write_files:
     #!/bin/bash
 
     set -ex
-    MASTERIP=`hostname -I | col1`
 
 
     echo "Launch HADOOP starts."
     echo 'Y' | /home/hduser/hadoop/bin/hdfs namenode -format hdfs_cluster
-    /home/hduser/hadoop/sbin/hadoop-daemon.sh start namenode
+    /home/hduser/hadoop/bin/hdfs --daemon start namenode
     echo "Launch HADOOP finished."
 
 
@@ -143,15 +141,10 @@ write_files:
    <configuration>
     <property>
      <name>dfs.namenode.http-address</name>
-      <value>HadoopMaster:50070</value>
-    </property>
-    <property>
-      <name>dfs.name.dir</name>
-      <value>/tmp</value>
-      <final>true</final>
+      <value>HadoopMaster:9870</value>
     </property>
     <property>
-       <name>dfs.permissions</name>
+       <name>dfs.permissions.enabled</name>
        <value>false</value>
     </property>
     <property>
@@ -181,46 +174,12 @@ write_files:
   content: |
       <configuration>
       <property>
-        <name>fs.default.name</name>
+        <name>fs.defaultFS</name>
         <value>hdfs://HadoopMaster:9000</value>
       </property>
       </configuration>
   permissions: '644'
 
-- path: /tmp/hadoop/webconfigs/web.xml
-  content: |
-    <?xml version="1.0" encoding="UTF-8"?>
-    <web-app version="2.4" xmlns="http://java.sun.com/xml/ns/j2ee">
-      <security-constraint>
-          <web-resource-collection>
-              <web-resource-name>Protected</web-resource-name>
-              <url-pattern>/*</url-pattern>
-          </web-resource-collection>
-          <auth-constraint>
-              <role-name>admin</role-name>
-          </auth-constraint>
-      </security-constraint>
-      <login-config>
-          <auth-method>BASIC</auth-method>
-          <realm-name>realm</realm-name>
-      </login-config>
-    </web-app>
-  permissions: '644'
-
-- path: /tmp/hadoop/webconfigs/jetty-web.xml
-  content: |
-    <Configure class="org.mortbay.jetty.webapp.WebAppContext">
-     <Get name="securityHandler">
-      <Set name="userRealm">
-        <New class="org.mortbay.jetty.security.HashUserRealm">
-          <Set name="name">realm</Set>
-          <Set name="config">/home/hduser/hadoop/etc/hadoop/realm.properties</Set>
-        </New>
-      </Set>
-     </Get>
-    </Configure>
-  permissions: '644'
-
 - path: /home/hduser/consul/hosts.ctmpl
   content: |
     127.0.0.1       localhost
@@ -296,8 +255,8 @@ write_files:
 
     if [ `diff /etc/hosts $FILE_LOCATION | grep '>' | wc -l` -gt 0 ]; then
         echo -e `date +%Y-%m-%d` `date +"%T"` "Downscale detected. Restarting name node service..."
-        su - hduser -c '/home/hduser/hadoop/sbin/hadoop-daemon.sh stop namenode'
-        su - hduser -c '/home/hduser/hadoop/sbin/hadoop-daemon.sh start namenode'
+        su - hduser -c '/home/hduser/hadoop/bin/hdfs --daemon stop namenode'
+        su - hduser -c '/home/hduser/hadoop/bin/hdfs --daemon start namenode'
         echo -e `date +%Y-%m-%d` `date +"%T"` "Namenode restarted!"
         cp /etc/hosts $FILE_LOCATION
     else [ `diff /etc/hosts $FILE_LOCATION | grep '<' | wc -l` -gt 0 ]
@@ -374,3 +333,4 @@ write_files:
 
 runcmd:
 - /tmp/installation.sh && /tmp/configuration.sh && su - hduser -c '/tmp/start-services.sh' && echo "HADOOP MASTER DEPLOYMENT DONE." || echo -e "\e[91mPROBLEM OCCURED WITH THE INSTALLATION\e[39m"
+
diff --git a/tutorials/hadoop-cluster/nodes/cloud_init_hadoop_slave.yaml b/tutorials/hadoop-cluster/nodes/cloud_init_hadoop_slave.yaml
@@ -32,7 +32,7 @@ write_files:
 
     export DEBIAN_FRONTEND=noninteractive
     apt-get update
-    apt-get install -y openjdk-8-jdk openjdk-8-jre unzip
+    apt-get install -yq --no-install-recommends openjdk-8-jdk openjdk-8-jre unzip
     echo "Install requirement packages starts."
 
 
@@ -110,11 +110,10 @@ write_files:
     #!/bin/bash
 
     set -ex
-    MASTERIP=`hostname -I | col1`
 
 
     echo "Launch HADOOP starts."
-    /home/hduser/hadoop/sbin/hadoop-daemon.sh start datanode
+    /home/hduser/hadoop/bin/hdfs --daemon start datanode
     echo "Launch HADOOP finished."
 
 
@@ -128,21 +127,20 @@ write_files:
    <configuration>
     <property>
      <name>dfs.namenode.http-address</name>
-      <value>{{getprivip('hadoop-master')}}:50070</value>
-    </property>
-    <property>
-      <name>dfs.name.dir</name>
-      <value>/tmp</value>
-      <final>true</final>
+      <value>{{getprivip('hadoop-master')}}:9870</value>
     </property>
     <property>
-       <name>dfs.permissions</name>
+       <name>dfs.permissions.enabled</name>
        <value>false</value>
     </property>
     <property>
       <name>dfs.datanode.du.reserved</name>
       <value>500000000</value>
     </property>
+    <property>
+      <name>dfs.hosts.exclude</name>
+      <value>/home/hduser/hadoop/etc/hadoop/dfs.exclude</value>
+    </property>
     <property>
       <name>dfs.client.use.datanode.hostname</name>
       <value>true</value>
@@ -162,7 +160,7 @@ write_files:
   content: |
       <configuration>
       <property>
-        <name>fs.default.name</name>
+        <name>fs.defaultFS</name>
         <value>hdfs://{{getprivip('hadoop-master')}}:9000</value>
       </property>
       </configuration>
@@ -232,4 +230,5 @@ write_files:
   permissions: '644'
 
 runcmd:
-- /tmp/installation.sh && /tmp/configuration.sh && su - hduser -c '/tmp/start-services.sh' && echo "HADOOP SLAVE DEPLOYMENT DONE." || echo -e "\e[91mPROBLEM OCCURED WITH THE INSTALLATION\e[39m"
+- /tmp/installation.sh && /tmp/configuration.sh && su - hduser -c '/tmp/start-services.sh' && echo "HADOOP SLAVE DEPLOYMENT DONE." || echo -e "\e[91mPROBLEM OCCURED WITH THE INSTALLATION\e[39m"
+
diff --git a/tutorials/hadoop-cluster/nodes/node_definitions.yaml b/tutorials/hadoop-cluster/nodes/node_definitions.yaml
@@ -20,8 +20,8 @@
         url: file://cloud_init_hadoop_master.yaml
     health_check:
       ports:
-        - 50070
-      timeout: 6000
+       - 9870
+      timeout: 1000
 
 'node_def:hadoop_slave_node':
   -
@@ -41,3 +41,6 @@
       type: cloudinit
       context_template: !yaml_import
         url: file://cloud_init_hadoop_slave.yaml
+    health_check:
+      timeout: 1000
+