Permalink
Browse files

[cm] Improvements & Bug Fixes

  • Loading branch information...
1 parent d21c745 commit a1e4808a12d79b6df1da548f1d2a7a4f9169173a @ashrithr ashrithr committed Apr 6, 2014
@@ -1,5 +1,5 @@
# Adds a host to existing cluster
-define cm::api::parcels::add_host_to_cluster {
+define cm::api::add_host_to_cluster {
require cm::api::params
curl { "add-host-${name}-to-cluster":
View
@@ -2,9 +2,14 @@
# find out the api version to work with
$cluster_name = $::cm::params::cm_cluster_name
$cm_domain = "http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}"
- $api_version = inline_template("<%= require 'uri'
- open(\"${cm_domain}/api/version\",
- :http_basic_authentication => [\"${::cm::params::cm_username}\", \"${::cm::params::cm_password}\"]
- ).string
+ $api_version = inline_template("<%=
+ require 'uri'
+ begin
+ open(\"${cm_domain}/api/version\",
+ :http_basic_authentication => [\"${::cm::params::cm_username}\", \"${::cm::params::cm_password}\"]
+ ).string
+ resuce OpenURI::HTTPError
+ nil
+ end
-%>")
}
@@ -2,31 +2,36 @@
require cm::api::params
$cluster_name = $cm::params::cm_cluster_name
- $parcel_version = inline_template("<%=
+ $activate_parcel_version = inline_template("<%=
require 'rubygems'
require 'json'
require 'open-uri'
- JSON.parse(open(\"${cm::api::params::cm_domain}/api/${cm::api::params::api_version}/clusters/${cluster_name}/parcels\",
- :http_basic_authentication => [\"${cm::params::cm_username}\", \"${cm::params::cm_password}\"]).string)['items'].map do |i|
- i['version'] if(i['product'] == '${product_name}' && i['version'] =~ /${product_version}/)
- end.compact!.first
+ begin
+ JSON.parse(open(\"${cm::api::params::cm_domain}/api/${cm::api::params::api_version}/clusters/${cluster_name}/parcels\",
+ :http_basic_authentication => [\"${cm::params::cm_username}\", \"${cm::params::cm_password}\"]).string)['items'].map do |i|
+ i['version'] if(i['product'] == '${product_name}' && i['version'] =~ /${product_version}/)
+ end.compact!.first
+ rescue OpenURI::HTTPError
+ nil
+ end
%>")
# Distribute parcels
curl { "activate-parcel-${product_name}-${product_version}":
domain => "http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}",
- post => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}/commands/activate",
+ post => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${activate_parcel_version}/commands/activate",
parameters => "{}",
request_type => 'json',
username => $::cm::params::cm_username,
password => $::cm::params::cm_password,
returns => '200',
only_if => {
- 'get' => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}",
+ 'get' => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${activate_parcel_version}",
'returns' => '200',
'contains_key' => "stage",
'contains_value' => 'DISTRIBUTED'
},
- log_to => "/var/log/ankus/cm_api/parcels_${name}"
+ log_to => "/var/log/ankus/cm_api/parcels_${name}",
+ require => Wait_for["${product_name}-${product_version} parcel distribution completes"]
}
}
@@ -4,24 +4,21 @@
cm::api::parcels::download { 'Download CDH':
product_name => 'CDH',
product_version => '4'
- } ->
+ }
cm::api::parcels::download { 'Download IMPALA':
product_name => 'IMPALA'
- } ->
- # add hosts to the cluster specified
- cm::api::parcels::add_host_to_cluster { $::cm::params::cm_nodes:
- } ->
+ }
cm::api::parcels::distribute { 'Distribute CDH':
product_name => 'CDH',
product_version => '4'
- } ->
+ }
cm::api::parcels::distribute { 'Distribute IMPALA':
product_name => 'IMPALA'
- } ->
+ }
cm::api::parcels::activate { 'Activate CDH':
product_name => 'CDH',
product_version => '4'
- } ->
+ }
cm::api::parcels::activate { 'Distribute IMPALA':
product_name => 'IMPALA'
}
@@ -2,37 +2,42 @@
require cm::api::params
$cluster_name = $cm::params::cm_cluster_name
- $parcel_version = inline_template("<%=
+ $distribute_parcel_version = inline_template("<%=
require 'rubygems'
require 'json'
require 'open-uri'
- JSON.parse(open(\"${cm::api::params::cm_domain}/api/${cm::api::params::api_version}/clusters/${cluster_name}/parcels\",
- :http_basic_authentication => [\"${cm::params::cm_username}\", \"${cm::params::cm_password}\"]).string)['items'].map do |i|
- i['version'] if(i['product'] == '${product_name}' && i['version'] =~ /${product_version}/)
- end.compact!.first
+ begin
+ JSON.parse(open(\"${cm::api::params::cm_domain}/api/${cm::api::params::api_version}/clusters/${cluster_name}/parcels\",
+ :http_basic_authentication => [\"${cm::params::cm_username}\", \"${cm::params::cm_password}\"]).string)['items'].map do |i|
+ i['version'] if(i['product'] == '${product_name}' && i['version'] =~ /${product_version}/)
+ end.compact!.first
+ rescue OpenURI::HTTPError
+ nil
+ end
%>")
# Distribute parcels
curl { "distribute-parcel-${product_name}-${product_version}":
domain => "http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}",
- post => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}/commands/startDistribution",
+ post => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${distribute_parcel_version}/commands/startDistribution",
parameters => "{}",
request_type => 'json',
username => $::cm::params::cm_username,
password => $::cm::params::cm_password,
returns => '200',
only_if => {
- 'get' => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}",
+ 'get' => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${distribute_parcel_version}",
'returns' => '200',
'contains_key' => "stage",
'contains_value' => 'DOWNLOADED'
},
- log_to => "/var/log/ankus/cm_api/parcels_${name}"
+ log_to => "/var/log/ankus/cm_api/parcels_${name}",
+ require => Wait_for["${product_name}-${product_version} parcel download completes"]
}
# wait until parcel download get's completed
wait_for { "${product_name}-${product_version} parcel distribution completes":
- query => "/usr/bin/curl --silent -X GET -u \"${::cm::params::cm_username}:${::cm::params::cm_password}\" -i -H \"content-type:application/json\" http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}",
+ query => "/usr/bin/curl --silent -X GET -u \"${::cm::params::cm_username}:${::cm::params::cm_password}\" -i -H \"content-type:application/json\" http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${distribute_parcel_version}",
regex => '.*stage.*:.*DISTRIBUTED.*',
giveup_regex => '.*stage.*:.*ACTIVATED.*',
polling_frequency => 15,
@@ -3,27 +3,31 @@
require cm::api::params
$cluster_name = $cm::params::cm_cluster_name
- $parcel_version = inline_template("<%=
+ $download_parcel_version = inline_template("<%=
require 'rubygems'
require 'json'
require 'open-uri'
- JSON.parse(open(\"${cm::api::params::cm_domain}/api/${cm::api::params::api_version}/clusters/${cluster_name}/parcels\",
- :http_basic_authentication => [\"${cm::params::cm_username}\", \"${cm::params::cm_password}\"]).string)['items'].map do |i|
- i['version'] if(i['product'] == '${product_name}' && i['version'] =~ /${product_version}/)
- end.compact!.first
+ begin
+ JSON.parse(open(\"${cm::api::params::cm_domain}/api/${cm::api::params::api_version}/clusters/${cluster_name}/parcels\",
+ :http_basic_authentication => [\"${cm::params::cm_username}\", \"${cm::params::cm_password}\"]).string)['items'].map do |i|
+ i['version'] if(i['product'] == '${product_name}' && i['version'] =~ /${product_version}/)
+ end.compact!.first
+ rescue OpenURI::HTTPError
+ nil
+ end
%>")
# Issue download
curl { "download-parcel-${product_name}-${product_version}":
domain => "http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}",
- post => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}/commands/startDownload",
+ post => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${download_parcel_version}/commands/startDownload",
parameters => "{}",
request_type => 'json',
username => $::cm::params::cm_username,
password => $::cm::params::cm_password,
returns => '200',
only_if => {
- 'get' => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}",
+ 'get' => "/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${download_parcel_version}",
'returns' => '200',
'contains_key' => "stage",
'contains_value' => 'AVAILABLE_REMOTELY'
@@ -33,7 +37,7 @@
# wait until parcel download get's completed
wait_for { "${product_name}-${product_version} parcel download completes":
- query => "/usr/bin/curl --silent -X GET -u \"${::cm::params::cm_username}:${::cm::params::cm_password}\" -i -H \"content-type:application/json\" http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${parcel_version}",
+ query => "/usr/bin/curl --silent -X GET -u \"${::cm::params::cm_username}:${::cm::params::cm_password}\" -i -H \"content-type:application/json\" http://${::cm::params::cm_server_host}:${::cm::params::cm_api_port}/api/${::cm::api::params::api_version}/clusters/${cluster_name}/parcels/products/${product_name}/versions/${download_parcel_version}",
regex => '.*stage.*:.*DOWNLOADED.*',
giveup_regex => '.*stage.*:.*DISTRIBUTED.*|.*stage.*:.*ACTIVATED.*',
polling_frequency => 15,
@@ -0,0 +1 @@
+# Run a command and monitor for it to complete
No changes.
View
@@ -0,0 +1,16 @@
+curl { 'test':
+ domain => "http://localhost:7180",
+ get => "/api/v6/clusters/test_cluster/parcels/products/CDH/versions/4.6.0-1.cdh4.6.0.p0.26",
+ parameters => "{}",
+ request_type => 'json',
+ username => "admin",
+ password => "admin",
+ returns => '200',
+ only_if => {
+ 'get' => "/api/v6/clusters/test_cluster/parcels/products/CDH/versions/4.6.0-1.cdh4.6.0.p0.26",
+ 'returns' => '200',
+ 'contains_key' => "stage",
+ 'contains_value' => 'AVAILABLE_REMOTELY'
+ },
+ log_to => '/var/log/ankus/cm_api/test'
+}
@@ -2,4 +2,14 @@
cm::api::cluster { $cm::params::cm_cluster_name:
cluster_version => $cm::params::cm_cluster_ver
}
+
+ # Add all the hosts to the cluster
+ cm::api::add_host_to_cluster { $::cm::params::cm_nodes:
+ require => Cm::Api::Cluster[$cm::params::cm_cluster_name]
+ }
+
+ # Configure parcels for the cluster
+ class { 'cm::api::parcels::configure':
+ require => Cm::Api::Add_host_to_cluster[$::cm::params::cm_nodes]
+ }
}
View
@@ -1,11 +1,12 @@
# Initializes & configures hdfs
class cm::hdfs_init inherits cm::params {
$service_name = 'hdfs-service'
- # intialize service
+ # intialize hdfs service
cm::api::service { $service_name:
service_type => 'HDFS',
+ require => Cm::Api::Parcels::Activate['Activate CDH']
} ->
- # configure root service
+ # configure root hdfs service
cm::api::service_config { ['hadoop_security_authentication', 'hadoop_security_authorization', 'dfs_block_size', 'dfs_replication_min', 'dfs_replication', 'dfs_permissions']:
service_name => $service_name,
meta_data => {
@@ -17,9 +18,23 @@
'dfs_permissions' => 'true'
},
} ->
- cm::api::role { 'namenode001':
+ # assign namenode role
+ $nn_role = 'namenode001'
+ cm::api::role { $nn_role:
service_name => $service_name,
role_type => 'NAMENODE',
host_id => $cm::params::namenode_host, # fqdn of the namende host
- }
+ } ->
+ # configure namenode role
+ cm::api::role_config { ['dfs_name_dir_list']:
+ service_name => $service_name,
+ role_name => $nn_role,
+ meta_data => {
+ 'dfs_name_dir_list' => $cm::params::dfs_name_dir,
+ },
+ } ->
+ # format namenode
+
+ # start namenode role
+
}
View
@@ -28,28 +28,19 @@
# Copyright 2012 Cloudwick Technologies, unless otherwise noted.
#
class cm($role, $pass = 1, $hdfs_enabled = true) {
- if ($pass == 1) {
- if ($role == 'server') {
- class { 'cm::server': } ->
- class { 'cm::agent': }
- } else {
- class { 'cm::agent': }
- }
- } elsif ($pass == 2) {
- if ($role == 'server') {
- class { 'cm::server': } ->
- class { 'cm::agent': } ->
- cm::api::cluster { $cm::params::cm_cluster_name:
- cluster_version => $cm::params::cm_cluster_ver
- } ->
- class { 'cm::api::parcels::configure': }
- if ($hdfs_enabled == true) {
- class { 'cm::hdfs_init':
- require => Class['cm::api::parcels::configure']
- }
+ if ($role == 'server' and $pass == 1) {
+ class { 'cm::server': } ->
+ class { 'cm::agent': }
+ } elsif ($role == 'server' and $pass == 2) {
+ class { 'cm::server': } ->
+ class { 'cm::agent': } ->
+ class { 'cm::cluster_init': }
+ if ($hdfs_enabled == true) {
+ class { 'cm::hdfs_init':
+ require => Class['cm::cluster_init']
}
- } else {
- class { 'cm::agent': }
}
+ } elsif ($role == 'agent') {
+ class { 'cm::agent': }
}
}
View
@@ -44,7 +44,7 @@
/4/ => 'cloudera-manager-server-db',
}
- $cm_server_host = hiera('cloudera_cm_server_host', 'localhost')
+ $cm_server_host = hiera('cloudera_cm_server_host', $::fqdn)
$cm_server_port = hiera('cloudera_cm_server_port', '7182')
@@ -103,4 +103,7 @@
$cm_hmon_listen_port = hiera('cm_hmon_listen_port', '9995')
$cm_hmon_database_type = hiera('cm_hmon_database_type', 'postgresql')
$cm_hmon_database_password = hiera('cm_hmon_database_password', 'hmon')
+
+ # Host specific
+ $namenode_host = hiera('hadoop_namenode', $::fqdn)
}
View
@@ -102,7 +102,7 @@
hasstatus => true,
require => Package['cloudera-manager-server'],
}
-
+
# package { $::cm::params::cm_embedded_database_pkg:
# ensure => $package_ensure,
# require => Class[$::cm::params::repo_class]

0 comments on commit a1e4808

Please sign in to comment.