Add in HDFS storage-tiering

.gitignore

@@ -20,6 +20,7 @@ cookbooks/*/
 !cookbooks/bcpc_jmxtrans
 !cookbooks/hannibal
 !cookbooks/bcpc_kafka
+!cookbooks/hdfsdu
 !cookbooks/bach_krb5
 !cookbooks/smoke-tests
 cookbooks/*/coverage

Berksfile

@@ -14,6 +14,7 @@ cookbook 'bcpc', path: './cookbooks/bcpc'
 cookbook 'bcpc-hadoop', path: './cookbooks/bcpc-hadoop'
 cookbook 'bcpc_jmxtrans', path: './cookbooks/bcpc_jmxtrans'
 cookbook 'hannibal', path: './cookbooks/hannibal'
+cookbook 'hdfsdu', path: './cookbooks/hdfsdu'
 cookbook 'bcpc_kafka', path: './cookbooks/bcpc_kafka'
 cookbook 'smoke-tests', path: './cookbooks/smoke-tests'
 

cookbooks/bach_repository/attributes/default.rb

@@ -71,3 +71,7 @@
 
 # Set the JAVA_HOME for Hadoop components
 default['bcpc']['hadoop']['java'] = "/usr/lib/jvm/java-8-oracle-amd64"
+
+# jmxtrans-agent (https://github.com/jmxtrans/jmxtrans-agent)
+default['bach']['repository']['jmxtrans_agent']['download_url'] = 'https://github.com/jmxtrans/jmxtrans-agent/releases/download/jmxtrans-agent-1.2.5/jmxtrans-agent-1.2.5.jar'
+default['bach']['repository']['jmxtrans_agent']['checksum'] = 'd351ac0b863ffb2742477001296f65cbca6f8e9bb5bec3dc2194c447d838ae17'

cookbooks/bach_repository/recipes/default.rb

@@ -36,5 +36,8 @@
 # build jvmkill lib
 include_recipe 'bach_repository::jvmkill'
 
+# download jmxtrans-agent
+include_recipe 'bach_repository::jmxtrans_agent'
+
 # Run after everything to fix perms.
 include_recipe 'bach_repository::permissions'

cookbooks/bach_repository/recipes/jmxtrans_agent.rb

@@ -0,0 +1,16 @@
+#
+# Cookbook Name:: bach_repository
+# Recipe:: jmxtrans_agent
+#
+include_recipe 'bach_repository::directory'
+bins_dir = node['bach']['repository']['bins_directory']
+target_file = bins_dir + '/jmxtrans-agent-1.2.5.jar'
+
+remote_file target_file do
+  source node['bach']['repository']['jmxtrans_agent']['download_url']
+  user 'root'
+  group 'root'
+  mode 0444
+  checksum node['bach']['repository']['jmxtrans_agent']['checksum']
+  not_if { File.exist?(target_file) }
+end

cookbooks/bcpc-hadoop/attributes/default.rb

@@ -6,12 +6,12 @@
 require 'pathname'
 
 user = node['bcpc']['bootstrap']['admin']['user']
+default['bcpc']['cluster']['file_path'] = "/home/#{user}/chef-bcpc/cluster.txt"
 
 default['bcpc']['hadoop'] = {}
 default['bcpc']['hadoop']['distribution']['release'] = '2.6.1.17-1'
 default['bcpc']['hadoop']['distribution']['active_release'] = node['bcpc']['hadoop']['distribution']['release']
 default['bcpc']['hadoop']['decommission']['hosts'] = []
-# disks to use for Hadoop activities (expected to be an environment or role set variable)
 default['bcpc']['hadoop']['hadoop_home_warn_suppress'] = 1
 default['bcpc']['hadoop']['hadoop_log_dir'] = '/var/log/hadoop-hdfs'
 default['bcpc']['hadoop']['hadoop_mapred_ident_string'] = 'mapred'
@@ -26,19 +26,14 @@
 # Flag to control whether automatic restarts due to config changes need to be skipped
 # for e.g. if ZK quorum is down or if the recipes need to be run in a non ZK env
 default['bcpc']['hadoop']['skip_restart_coordination'] = false
-default['bcpc']['hadoop']['hdfs']['site_xml']['dfs.datanode.sync.behind.writes'] = true
-default['bcpc']['hadoop']['hdfs']['site_xml']['dfs.datanode.synconclose'] = true
-default['bcpc']['hadoop']['hdfs']['site_xml']['dfs.namenode.stale.datanode.interval'] = 30_000
 default['bcpc']['hadoop']['hdfs']['HA'] = true
 default['bcpc']['hadoop']['hdfs']['failed_volumes_tolerated'] = 1
 default['bcpc']['hadoop']['hdfs']['dfs_replication_factor'] = 3
 default['bcpc']['hadoop']['hdfs']['dfs_blocksize'] = '128m'
 default['bcpc']['hadoop']['hdfs_url'] = "hdfs://#{node.chef_environment}"
-default['bcpc']['hadoop']['jmx_enabled'] = true
-default[:bcpc][:hadoop][:jute][:maxbuffer] = 6_291_456
-default['bcpc']['hadoop']['datanode']['xmx']['max_size'] = 4_096
-default['bcpc']['hadoop']['datanode']['xmx']['max_ratio'] = 0.25
-default['bcpc']['hadoop']['datanode']['max']['xferthreads'] = 16_384
+default['bcpc']['hadoop']['jmx_enabled'] = false
+default['bcpc']['hadoop']['jmx_agent_enabled'] = true
+default['bcpc']['hadoop']['java_https_keystore'] = '/usr/local/maven/conf/keystore'
 
 # for jvmkill library
 default['bcpc-hadoop']['jvmkill']['lib_file'] = '/var/lib/jvmkill/libjvmkill.so'
@@ -77,15 +72,6 @@
   common_opts
 
 default['bcpc']['hadoop']['mapreduce']['framework']['name'] = 'yarn'
-default['bcpc']['hadoop']['namenode']['handler']['count'] = 100
-# set to nil to calculate dynamically based on available memory
-default['bcpc']['hadoop']['namenode']['xmx']['max_size'] = 1024
-# set to nil to calculate dynamically based on available memory
-default['bcpc']['hadoop']['namenode']['xmn']['max_size'] = 128
-default['bcpc']['hadoop']['namenode']['xmx']['max_ratio'] = 0.25
-default['bcpc']['hadoop']['namenode']['rpc']['port'] = 8020
-default['bcpc']['hadoop']['namenode']['http']['port'] = 50070
-default['bcpc']['hadoop']['namenode']['https']['port'] = 50470
 default['bcpc']['hadoop']['kafka']['jmx']['port'] = 9995
 default['bcpc']['hadoop']['topology']['script'] = 'topology'
 default['bcpc']['hadoop']['topology']['cookbook'] = 'bcpc-hadoop'
@@ -209,5 +195,3 @@
 
 # Set the JAVA_HOME for Hadoop components
 default['bcpc']['hadoop']['java'] = '/usr/lib/jvm/java-8-oracle-amd64'
-
-default['bcpc']['cluster']['file_path'] = "/home/#{user}/chef-bcpc/cluster.txt"

cookbooks/bcpc-hadoop/attributes/hdfs.rb

@@ -1,9 +1,23 @@
-# vim: tabstop=2:shiftwidth=2:softtabstop=2 
+# vim: tabstop=2:shiftwidth=2:softtabstop=2
 # Setting balancer andwidth to default value as per hdfs-default.xml
 default["hadoop"]["hdfs"]["balancer"]["bandwidth"] = 1048576
 # balancer thread multiplier constant
 default["hadoop"]["hdfs"]["balancer"]["max_concurrent_moves_multiplier"] = 10
 
+default['bcpc']['hadoop']['datanode']['xmx']['max_size'] = 4_096
+default['bcpc']['hadoop']['datanode']['xmx']['max_ratio'] = 0.25
+default['bcpc']['hadoop']['datanode']['max']['xferthreads'] = 16_384
+default['bcpc']['hadoop']['namenode']['handler']['count'] = 100
+# set to nil to calculate dynamically based on available memory
+default['bcpc']['hadoop']['namenode']['xmx']['max_size'] = 1024
+# # set to nil to calculate dynamically based on available memory
+default['bcpc']['hadoop']['namenode']['xmn']['max_size'] = 128
+default['bcpc']['hadoop']['namenode']['xmx']['max_ratio'] = 0.25
+default['bcpc']['hadoop']['namenode']['jmx']['port'] = 10111
+default['bcpc']['hadoop']['namenode']['rpc']['port'] = 8020
+default['bcpc']['hadoop']['namenode']['http']['port'] = 50070
+default['bcpc']['hadoop']['namenode']['https']['port'] = 50470
+
 # JMX port mappings
 default['bcpc']['hadoop'].tap do |jmx|
   jmx['journalnode']['jmx']['port'] = 10110
@@ -41,13 +55,16 @@
 
 default[:bcpc][:hadoop][:hdfs][:site_xml].tap do |site_xml|
   dfs = node[:bcpc][:hadoop][:hdfs][:dfs]
-  
+
   site_xml['dfs.replication'] =
     node[:bcpc][:hadoop][:hdfs][:dfs_replication_factor]
-  
+
   site_xml['dfs.namenode.audit.log.async'] =
     dfs[:namenode][:audit][:log][:async]
 
+  site_xml['dfs.datanode.sync.behind.writes'] = true
+  site_xml['dfs.datanode.synconclose'] = true
+  site_xml['dfs.namenode.stale.datanode.interval'] = 30_000
 
   site_xml['dfs.nameservices'] = node.chef_environment
 
@@ -70,7 +87,7 @@
 
   site_xml['dfs.client.read.shortcircuit'] =
     dfs[:client][:read][:shortcircuit]
-  
+
   site_xml['dfs.domain.socket.path'] =
     dfs[:domain][:socket][:path]
 
@@ -100,7 +117,7 @@
 
   site_xml['dfs.datanode.max.transfer.threads'] =
     node[:bcpc][:hadoop][:datanode][:max][:xferthreads]
-  
+
   site_xml['dfs.namenode.handler.count'] =
     node[:bcpc][:hadoop][:namenode][:handler][:count]