[cm] Improvements & Bug Fixes

cm/manifests/api/add_host_to_cluster.pp

@@ -1,5 +1,5 @@
 # Adds a host to existing cluster
-define cm::api::parcels::add_host_to_cluster {
+define cm::api::add_host_to_cluster {
   require cm::api::params
 
   curl { "add-host-${name}-to-cluster":

cm/manifests/api/params.pp

@@ -2,9 +2,14 @@
   # find out the api version to work with
   $cluster_name = $::cm::params::cm_cluster_name
   $cm_domain = "http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}"
-  $api_version = inline_template("<%= require 'uri'
-    open(\"${cm_domain}/api/version\",
-      :http_basic_authentication => [\"${::cm::params::cm_username}\", \"${::cm::params::cm_password}\"]
-    ).string
+  $api_version = inline_template("<%=
+    require 'uri'
+    begin
+      open(\"${cm_domain}/api/version\",
+        :http_basic_authentication => [\"${::cm::params::cm_username}\", \"${::cm::params::cm_password}\"]
+      ).string
+    resuce OpenURI::HTTPError
+      nil
+    end
   -%>")
 }

cm/manifests/api/parcels/activate.pp

@@ -2,31 +2,36 @@
   require cm::api::params
 
   $cluster_name   = $cm::params::cm_cluster_name
-  $parcel_version = inline_template("<%=
+  $activate_parcel_version = inline_template("<%=
     require 'rubygems'
     require 'json'
     require 'open-uri'
-    JSON.parse(open(\"${cm::api::params::cm_domain}/api/${cm::api::params::api_version}/clusters/${cluster_name}/parcels\",
-      :http_basic_authentication => [\"${cm::params::cm_username}\", \"${cm::params::cm_password}\"]).string)['items'].map do |i|
-        i['version'] if(i['product'] == '${product_name}' && i['version'] =~ /${product_version}/)
-    end.compact!.first
+    begin
+      JSON.parse(open(\"${cm::api::params::cm_domain}/api/${cm::api::params::api_version}/clusters/${cluster_name}/parcels\",
+        :http_basic_authentication => [\"${cm::params::cm_username}\", \"${cm::params::cm_password}\"]).string)['items'].map do |i|
+          i['version'] if(i['product'] == '${product_name}' && i['version'] =~ /${product_version}/)
+      end.compact!.first
+    rescue OpenURI::HTTPError
+      nil
+    end
   %>")
 
   # Distribute parcels
   curl { "activate-parcel-${product_name}-${product_version}":
     domain       => "http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}",
-    post         => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}/commands/activate",
+    post         => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${activate_parcel_version}/commands/activate",
     parameters   => "{}",
     request_type => 'json',
     username     => $::cm::params::cm_username,
     password     => $::cm::params::cm_password,
     returns      => '200',
     only_if      => {
-                      'get'            => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}",
+                      'get'            => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${activate_parcel_version}",
                       'returns'        => '200',
                       'contains_key'   => "stage",
                       'contains_value' => 'DISTRIBUTED'
                     },
-    log_to      => "/var/log/ankus/cm_api/parcels_${name}"
+    log_to      => "/var/log/ankus/cm_api/parcels_${name}",
+    require     => Wait_for["${product_name}-${product_version} parcel distribution completes"]
   }
 }

cm/manifests/api/parcels/configure.pp

@@ -4,24 +4,21 @@
   cm::api::parcels::download { 'Download CDH':
     product_name => 'CDH',
     product_version => '4'
-  } ->
+  }
   cm::api::parcels::download { 'Download IMPALA':
     product_name => 'IMPALA'
-  } ->
-  # add hosts to the cluster specified
-  cm::api::parcels::add_host_to_cluster { $::cm::params::cm_nodes:
-  } ->
+  }
   cm::api::parcels::distribute { 'Distribute CDH':
     product_name => 'CDH',
     product_version => '4'
-  } ->
+  }
   cm::api::parcels::distribute { 'Distribute IMPALA':
     product_name => 'IMPALA'
-  } ->
+  }
   cm::api::parcels::activate { 'Activate CDH':
     product_name => 'CDH',
     product_version => '4'
-  } ->
+  }
   cm::api::parcels::activate { 'Distribute IMPALA':
     product_name => 'IMPALA'
   }

cm/manifests/api/parcels/distribute.pp

@@ -2,37 +2,42 @@
   require cm::api::params
 
   $cluster_name   = $cm::params::cm_cluster_name
-  $parcel_version = inline_template("<%=
+  $distribute_parcel_version = inline_template("<%=
     require 'rubygems'
     require 'json'
     require 'open-uri'
-    JSON.parse(open(\"${cm::api::params::cm_domain}/api/${cm::api::params::api_version}/clusters/${cluster_name}/parcels\",
-      :http_basic_authentication => [\"${cm::params::cm_username}\", \"${cm::params::cm_password}\"]).string)['items'].map do |i|
-        i['version'] if(i['product'] == '${product_name}' && i['version'] =~ /${product_version}/)
-    end.compact!.first
+    begin
+      JSON.parse(open(\"${cm::api::params::cm_domain}/api/${cm::api::params::api_version}/clusters/${cluster_name}/parcels\",
+        :http_basic_authentication => [\"${cm::params::cm_username}\", \"${cm::params::cm_password}\"]).string)['items'].map do |i|
+          i['version'] if(i['product'] == '${product_name}' && i['version'] =~ /${product_version}/)
+      end.compact!.first
+    rescue OpenURI::HTTPError
+      nil
+    end
   %>")
 
   # Distribute parcels
   curl { "distribute-parcel-${product_name}-${product_version}":
     domain       => "http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}",
-    post         => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}/commands/startDistribution",
+    post         => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${distribute_parcel_version}/commands/startDistribution",
     parameters   => "{}",
     request_type => 'json',
     username     => $::cm::params::cm_username,
     password     => $::cm::params::cm_password,
     returns      => '200',
     only_if      => {
-                      'get'            => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}",
+                      'get'            => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${distribute_parcel_version}",
                       'returns'        => '200',
                       'contains_key'   => "stage",
                       'contains_value' => 'DOWNLOADED'
                     },
-    log_to      => "/var/log/ankus/cm_api/parcels_${name}"
+    log_to      => "/var/log/ankus/cm_api/parcels_${name}",
+    require     => Wait_for["${product_name}-${product_version} parcel download completes"]
   }
 
   # wait until parcel download get's completed
   wait_for { "${product_name}-${product_version} parcel distribution completes":
-    query             => "/usr/bin/curl --silent -X GET -u \"${::cm::params::cm_username}:${::cm::params::cm_password}\" -i -H \"content-type:application/json\" http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}",
+    query             => "/usr/bin/curl --silent -X GET -u \"${::cm::params::cm_username}:${::cm::params::cm_password}\" -i -H \"content-type:application/json\" http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${distribute_parcel_version}",
     regex             => '.*stage.*:.*DISTRIBUTED.*',
     giveup_regex      => '.*stage.*:.*ACTIVATED.*',
     polling_frequency => 15,

cm/manifests/api/parcels/download.pp

@@ -3,27 +3,31 @@
   require cm::api::params
 
   $cluster_name   = $cm::params::cm_cluster_name
-  $parcel_version = inline_template("<%=
+  $download_parcel_version = inline_template("<%=
     require 'rubygems'
     require 'json'
     require 'open-uri'
-    JSON.parse(open(\"${cm::api::params::cm_domain}/api/${cm::api::params::api_version}/clusters/${cluster_name}/parcels\",
-      :http_basic_authentication => [\"${cm::params::cm_username}\", \"${cm::params::cm_password}\"]).string)['items'].map do |i|
-        i['version'] if(i['product'] == '${product_name}' && i['version'] =~ /${product_version}/)
-    end.compact!.first
+    begin
+      JSON.parse(open(\"${cm::api::params::cm_domain}/api/${cm::api::params::api_version}/clusters/${cluster_name}/parcels\",
+        :http_basic_authentication => [\"${cm::params::cm_username}\", \"${cm::params::cm_password}\"]).string)['items'].map do |i|
+          i['version'] if(i['product'] == '${product_name}' && i['version'] =~ /${product_version}/)
+      end.compact!.first
+    rescue OpenURI::HTTPError
+      nil
+    end
   %>")
 
   # Issue download
   curl { "download-parcel-${product_name}-${product_version}":
     domain       => "http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}",
-    post         => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}/commands/startDownload",
+    post         => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${download_parcel_version}/commands/startDownload",
     parameters   => "{}",
     request_type => 'json',
     username     => $::cm::params::cm_username,
     password     => $::cm::params::cm_password,
     returns      => '200',
     only_if      => {
-                      'get'            => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}",
+                      'get'            => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${download_parcel_version}",
                       'returns'        => '200',
                       'contains_key'   => "stage",
                       'contains_value' => 'AVAILABLE_REMOTELY'
@@ -33,7 +37,7 @@
 
   # wait until parcel download get's completed
   wait_for { "${product_name}-${product_version} parcel download completes":
-    query             => "/usr/bin/curl --silent -X GET -u \"${::cm::params::cm_username}:${::cm::params::cm_password}\" -i -H \"content-type:application/json\" http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}",
+    query             => "/usr/bin/curl --silent -X GET -u \"${::cm::params::cm_username}:${::cm::params::cm_password}\" -i -H \"content-type:application/json\" http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${download_parcel_version}",
     regex             => '.*stage.*:.*DOWNLOADED.*',
     giveup_regex      => '.*stage.*:.*DISTRIBUTED.*|.*stage.*:.*ACTIVATED.*',
     polling_frequency => 15,

cm/manifests/api/run_command.pp

@@ -0,0 +1 @@
+# Run a command and monitor for it to complete

cm/manifests/api/test.pp

@@ -0,0 +1,16 @@
+curl { 'test':
+  domain       => "http://localhost:7180",
+  get         => "/api/v6/clusters/test_cluster/parcels/products/CDH/versions/4.6.0-1.cdh4.6.0.p0.26",
+  parameters   => "{}",
+  request_type => 'json',
+  username     => "admin",
+  password     => "admin",
+  returns      => '200',
+  only_if      => {
+                    'get' => "/api/v6/clusters/test_cluster/parcels/products/CDH/versions/4.6.0-1.cdh4.6.0.p0.26",
+                    'returns' => '200',
+                    'contains_key'   => "stage",
+                    'contains_value' => 'AVAILABLE_REMOTELY'
+                  },
+  log_to      => '/var/log/ankus/cm_api/test'
+}

cm/manifests/cluster_init.pp

@@ -2,4 +2,14 @@
   cm::api::cluster { $cm::params::cm_cluster_name:
     cluster_version => $cm::params::cm_cluster_ver
   }
+
+  # Add all the hosts to the cluster
+  cm::api::add_host_to_cluster { $::cm::params::cm_nodes:
+    require => Cm::Api::Cluster[$cm::params::cm_cluster_name]
+  }
+
+  # Configure parcels for the cluster
+  class { 'cm::api::parcels::configure':
+    require => Cm::Api::Add_host_to_cluster[$::cm::params::cm_nodes]
+  }
 }

cm/manifests/hdfs_init.pp

@@ -1,11 +1,12 @@
 # Initializes & configures hdfs
 class cm::hdfs_init inherits cm::params {
   $service_name = 'hdfs-service'
-  # intialize service
+  # intialize hdfs service
   cm::api::service { $service_name:
     service_type => 'HDFS',
+    require => Cm::Api::Parcels::Activate['Activate CDH']
   } ->
-  # configure root service
+  # configure root hdfs service
   cm::api::service_config { ['hadoop_security_authentication', 'hadoop_security_authorization', 'dfs_block_size', 'dfs_replication_min', 'dfs_replication', 'dfs_permissions']:
     service_name => $service_name,
     meta_data    => {
@@ -17,9 +18,23 @@
                       'dfs_permissions'                => 'true'
                     },
   } ->
-  cm::api::role { 'namenode001':
+  # assign namenode role
+  $nn_role = 'namenode001'
+  cm::api::role { $nn_role:
     service_name => $service_name,
     role_type    => 'NAMENODE',
     host_id      => $cm::params::namenode_host, # fqdn of the namende host
-  }
+  } ->
+  # configure namenode role
+  cm::api::role_config { ['dfs_name_dir_list']:
+    service_name => $service_name,
+    role_name => $nn_role,
+    meta_data => {
+                   'dfs_name_dir_list' => $cm::params::dfs_name_dir,
+                 },
+  } ->
+  # format namenode
+
+  # start namenode role
+
 }

cm/manifests/init.pp

@@ -28,28 +28,19 @@
 # Copyright 2012 Cloudwick Technologies, unless otherwise noted.
 #
 class cm($role, $pass = 1, $hdfs_enabled = true) {
-  if ($pass == 1) {
-    if ($role == 'server') {
-      class { 'cm::server': } ->
-      class { 'cm::agent': }
-    } else {
-      class { 'cm::agent': }
-    }
-  } elsif ($pass == 2) {
-    if ($role == 'server') {
-      class { 'cm::server': } ->
-      class { 'cm::agent': } ->
-      cm::api::cluster { $cm::params::cm_cluster_name:
-        cluster_version => $cm::params::cm_cluster_ver
-      } ->
-      class { 'cm::api::parcels::configure': }
-      if ($hdfs_enabled == true) {
-        class { 'cm::hdfs_init':
-          require => Class['cm::api::parcels::configure']
-        }
+  if ($role == 'server' and $pass == 1) {
+    class { 'cm::server': } ->
+    class { 'cm::agent': }
+  } elsif ($role == 'server' and $pass == 2) {
+    class { 'cm::server': } ->
+    class { 'cm::agent': } ->
+    class { 'cm::cluster_init': }
+    if ($hdfs_enabled == true) {
+      class { 'cm::hdfs_init':
+        require => Class['cm::cluster_init']
       }
-    } else {
-      class { 'cm::agent': }
     }
+  } elsif ($role == 'agent') {
+    class { 'cm::agent': }
   }
 }

cm/manifests/params.pp

@@ -44,7 +44,7 @@
     /4/ => 'cloudera-manager-server-db',
   }
 
-  $cm_server_host = hiera('cloudera_cm_server_host', 'localhost')
+  $cm_server_host = hiera('cloudera_cm_server_host', $::fqdn)
 
   $cm_server_port = hiera('cloudera_cm_server_port', '7182')
 
@@ -103,4 +103,7 @@
   $cm_hmon_listen_port       = hiera('cm_hmon_listen_port', '9995')
   $cm_hmon_database_type     = hiera('cm_hmon_database_type', 'postgresql')
   $cm_hmon_database_password = hiera('cm_hmon_database_password', 'hmon')
+
+  # Host specific
+  $namenode_host = hiera('hadoop_namenode', $::fqdn)
 }

cm/manifests/server.pp

@@ -102,7 +102,7 @@
     hasstatus  => true,
     require    => Package['cloudera-manager-server'],
   }
-
+  
   # package { $::cm::params::cm_embedded_database_pkg:
   #   ensure  => $package_ensure,
   #   require => Class[$::cm::params::repo_class]