UEFI and Ubuntu 14.04 support for chef-bach

 - First attempt at updating shell scripts for Ubuntu 14.04
 - Make cobbler recipe pass rubocop
 - Remove hardcoded paths from gem installation recipes.
 - Update bcpc::ufw recipe for 14.04 and rubocop compliance
 - Update bcpc::apache-mirror for 14.04
 - Update dependencies in metadata.rb and Berksfile to match shell scripts
 - Add .chef_found* to chefignore
 - Use upstream packages and cobbler 2.6
 - Use os_version 'trusty' for ubuntu 14.04 image import
 - Update enroll_cobbler.sh for cobbler 2.6
 - Use Xenial HWE kernel for EFI installs; reorganize cobbler templates.
 - Add 14.04 preseed, remove 12.04 preseed
 - Correct whitespace on bcpc default attributes
 - Fix vbox_create.sh for VM builds with EFI (again)
 - Don't write empty strings to environment if no proxy specified
 - automated_install.sh: Don't overwrite host's cluster dir in VM builds
 - Override data disks in the stub environment for VM builds
 - Move serial console kopts from enroll_cobbler.sh into bcpc::cobbler
 - Use the -56 percona metapackage instead of the versionless one
 - Updated quorum status check not to use mysql cli
 - Fix disk handling for EFI VM builds
 - Move percona repo to bcpc::mysql_client recipe
 - Delete a couple of bogus methods from bcpc::hadoop utils.rb
 - Fix mysql logging
 - Improve rubocop compliance on mysql.rb
 - Rubocop-ize bcpc-hadoop::oozie
 - Move oozie recipes from shell scripts to database cookbook providers
 - Move mysql connection info into helpers
 - Don't create ipxe hardlink until cobbler sync runs.
 - Straighten out tftp paths on bcpc::cobbler
 - Delete (duplicate) defaults from Test-Laptop demo environment
 - Grant global privileges to root@'%' in bcpc::mysql
 - Correct root@'%' priveleges; memoize passwords in connection info

.gitignore

@@ -20,5 +20,6 @@ cookbooks/*/
 !cookbooks/hannibal
 !cookbooks/kafka-bcpc
 !cookbooks/bach_krb5
+!cookbooks/bach_repository
 cookbooks/*/coverage
 cookbooks/*/vendor

Berksfile

@@ -46,7 +46,6 @@ cookbook 'ubuntu'
 cookbook 'bfd',
   git: 'https://github.com/bloomberg/openbfdd-cookbook'
 
-
 # cobblerd forked, pending destruction of earth by moon.
 cookbook 'cobblerd', 
   git: 'https://github.com/bloomberg/cobbler-cookbook'

Vagrantfile

@@ -91,7 +91,11 @@ Vagrant.configure('2') do |config|
                          netmask: '255.255.255.0',
                          adapter_ip: '192.168.100.2')
 
-    bootstrap.vm.synced_folder '../', '/chef-bcpc-host'
+    if File.basename(File.expand_path('.')) == 'vbox'
+      bootstrap.vm.synced_folder '../', '/chef-bcpc-host'
+    else
+      bootstrap.vm.synced_folder './', '/chef-bcpc-host'
+    end
 
     # set up repositories
     if $bach_local_mirror
@@ -103,8 +107,8 @@ Vagrant.configure('2') do |config|
     end
   end
 
-  config.vm.box = 'precise64'
-  config.vm.box_url = 'precise-server-cloudimg-amd64-vagrant-disk1.box'
+  config.vm.box = 'trusty64'
+  config.vm.box_url = 'trusty-server-cloudimg-amd64-vagrant-disk1.box'
 
   memory = ENV['BOOTSTRAP_VM_MEM'] || '1024'
   cpus = ENV['BOOTSTRAP_VM_CPUs'] || '1'

build_bins.sh

@@ -43,7 +43,7 @@ mkdir -p $APT_REPO_BINS
 apt-get -y update
 
 # Install tools needed for packaging
-apt-get -y install git rubygems make pkg-config pbuilder python-mock python-configobj python-support cdbs python-all-dev python-stdeb libmysqlclient-dev libldap2-dev ruby-dev gcc patch rake ruby1.9.3 ruby1.9.1-dev python-pip python-setuptools dpkg-dev apt-utils haveged libtool autoconf automake autotools-dev unzip rsync autogen
+apt-get -y install git ruby make pkg-config pbuilder python-mock python-configobj python-support cdbs python-all-dev python-stdeb libmysqlclient-dev libldap2-dev ruby-dev gcc patch rake ruby1.9.3 ruby1.9.1-dev python-pip python-setuptools dpkg-dev apt-utils haveged libtool autoconf automake autotools-dev unzip rsync autogen
 
 # Install json gem first to avoid a too-new version being pulled in by other gems.
 if [[ -z `gem list --local json | grep json | cut -f1 -d" "` ]]; then
@@ -94,21 +94,21 @@ if ! [[ -f jce_policy-8.zip ]]; then
 fi
 FILES="jce_policy-8.zip $FILES"
 
-# Pull all the gems required for the cluster 
-for i in patron wmi-lite simple-graphite ruby-augeas; do
+# Pull all the (unversioned) gems required for the cluster 
+for i in patron wmi-lite simple-graphite ruby-augeas chef-rewind; do
   if ! [[ -f gems/${i}.gem ]]; then
     gem fetch ${i}
     ln -s ${i}-*.gem ${i}.gem || true
   fi
   FILES="${i}*.gem $FILES"
 done
 
-# Get the Rubygem for zookeeper
-if ! [[ -f gems/zookeeper.gem ]]; then
-  gem fetch zookeeper -v 1.4.7
-  ln -s zookeeper-*.gem zookeeper.gem || true
+# Get the Rubygem for mysql2
+if ! [[ -f gems/mysql2.gem ]]; then
+  gem fetch mysql2 -v 0.4.4
+  ln -s mysql2-*.gem mysql2.gem || true
 fi
-FILES="zookeeper*.gem $FILES"
+FILES="mysql2*.gem $FILES"
 
 # Get the Rubygem for kerberos
 if ! [[ -f gems/rake-compiler.gem ]]; then
@@ -117,26 +117,40 @@ if ! [[ -f gems/rake-compiler.gem ]]; then
 fi
 FILES="rake-compiler*.gem $FILES"
 
-# Get the Rubygem for kerberos
+# Get the Rubygem for sequel
+if ! [[ -f gems/sequel.gem ]]; then
+  gem fetch sequel -v 4.36.0
+  ln -s sequel-*.gem sequel.gem || true
+fi
+FILES="sequel*.gem $FILES"
+
+# Get the Rubygem for rkerberos
 if ! [[ -f gems/rkerberos.gem ]]; then
   gem fetch rkerberos
   ln -s rkerberos*.gem rkerberos.gem || true
 fi
 FILES="rkerberos*.gem $FILES"
 
+# Get the Rubygem for webhdfs
+if ! [[ -f gems/webhdfs.gem ]]; then
+  gem fetch webhdfs -v 0.5.5
+  ln -s webhdfs-*.gem webhdfs.gem || true
+fi
+FILES="webhdfs*.gem $FILES"
+
 # Get Rubygem for zabbixapi
 if ! [[ -f gems/zabbixapi.gem ]]; then
   gem fetch zabbixapi -v ${ZABBIXAPI_VERSION}
   ln -s zabbix*.gem zabbixapi.gem || true
 fi
 FILES="zabbix*.gem $FILES"
 
-# Get the Rubygem for webhdfs
-if ! [[ -f gems/webhdfs.gem ]]; then
-  gem fetch webhdfs -v 0.5.5
-  ln -s webhdfs-*.gem webhdfs.gem || true
+# Get the Rubygem for zookeeper
+if ! [[ -f gems/zookeeper.gem ]]; then
+  gem fetch zookeeper -v 1.4.7
+  ln -s zookeeper-*.gem zookeeper.gem || true
 fi
-FILES="webhdfs*.gem $FILES"
+FILES="zookeeper*.gem $FILES"
 
 # Fetch the cirros image for testing
 if ! [[ -f cirros-0.3.0-x86_64-disk.img ]]; then
@@ -146,15 +160,12 @@ if ! [[ -f cirros-0.3.0-x86_64-disk.img ]]; then
 fi
 FILES="cirros-0.3.0-x86_64-disk.img $FILES"
 
-# Grab the Ubuntu 12.04 installer image
-UBUNTU_IMAGE="ubuntu-12.04-mini.iso"
-
+# Grab the Ubuntu 14.04 installer image with a 4.4 "HWE" kernel from Xenial
+UBUNTU_IMAGE="ubuntu-14.04-hwe44-mini.iso"
 if ! [[ -f $UBUNTU_IMAGE ]]; then
-  # Download this ISO to get the latest kernel/X LTS stack installer
-  #$CURL -o $UBUNTU_IMAGE http://archive.ubuntu.com/ubuntu/dists/precise-updates/main/installer-amd64/current/images/raring-netboot/mini.iso
-  while ! $(file $UBUNTU_IMAGE | grep -qE '(x86 boot sector)|(ISO 9660 CD-ROM)'); do
-    $CURL -o $UBUNTU_IMAGE http://archive.ubuntu.com/ubuntu/dists/precise-updates/main/installer-amd64/20101020ubuntu136.21/images/netboot/mini.iso
-  done
+    while ! $(file $UBUNTU_IMAGE | grep -qE '(x86 boot sector)|(ISO 9660 CD-ROM)'); do
+	$CURL -o $UBUNTU_IMAGE http://archive.ubuntu.com/ubuntu/dists/trusty-updates/main/installer-amd64/current/images/xenial-netboot/mini.iso
+    done
 fi
 FILES="$UBUNTU_IMAGE $FILES"
 
@@ -329,15 +340,24 @@ chmod -R 755 .
 
 ####################
 # generate Pypi repo
-
-# Wheel installs require setuptools >= 0.8 for dist-info support.
-# can then follow http://askubuntu.com/questions/399446
-# but can't upgrade setuptools first as:
-# "/usr/bin/pip install: error: no such option: --no-use-wheel"
 if ! hash dir2pi; then
-  /usr/bin/pip install pip2pi || /bin/true
-  /usr/local/bin/pip install setuptools --no-use-wheel --upgrade
-  /usr/local/bin/pip install pip2pi
+    PIP_VERSION=`pip --version | perl -nle 'm/pip\s+([\d\.]+)/; print $1'`
+
+    # If we have an ancient pip, upgrade it before getting pip2pi.
+    if ruby -e "exit 1 if Gem::Version.new('$PIP_VERSION') > \ 
+                          Gem::Version.new('1.5.4')"
+    then
+	# Wheel installs require setuptools >= 0.8 for dist-info support.
+	# can then follow http://askubuntu.com/questions/399446
+	# but can't upgrade setuptools first as:
+	# "/usr/bin/pip install: error: no such option: --no-use-wheel"
+	echo "Upgrading pip before pip2pi install"
+	/usr/bin/pip install pip2pi || /bin/true
+	/usr/local/bin/pip install setuptools --no-use-wheel --upgrade
+	/usr/local/bin/pip install pip2pi
+    else
+	/usr/bin/pip install pip2pi
+    fi	 
 fi
 
 dir2pi python
@@ -356,3 +376,4 @@ fi
 gem generate_index --legacy
 
 popd
+

chefignore

@@ -3,6 +3,7 @@
 .git
 .kitchen
 .vagrant
+bins
 chef
 cookbooks
 core

cookbooks/bach_krb5/recipes/gems.rb

@@ -1,10 +1,12 @@
 require 'pathname'
 require 'rubygems'
-gem_path = Pathname.new(Gem.ruby).dirname.join("gem").to_s
+require 'shellwords'
 
-gem_package "rake-compiler" do
+gem_path = Pathname.new(Gem.ruby).dirname.join('gem').to_s
+
+gem_package 'rake-compiler' do
   gem_binary gem_path
-  version ">=0.0.0"
+  version '>=0.0.0'
   action :nothing
 end.run_action(:install)
 
@@ -14,15 +16,23 @@
   end.run_action(:install)
 end
 
-gem_package "rkerberos" do
+gem_package 'rkerberos' do
   gem_binary gem_path
-  version ">=0.0.0"
+  version '>=0.0.0'
   action :nothing
 end.run_action(:install)
 
-execute "correct-gem-permissions" do
-  command 'find /opt/chef/embedded/lib/ruby/gems -type f -exec chmod a+r {} \; && ' +
-          'find /opt/chef/embedded/lib/ruby/gems -type d -exec chmod a+rx {} \;'
-  user "root"
+#
+# BACH typically runs chef-client with an abnormal umask, which causes
+# rubygems to install files with bad permissions.
+#
+# This execute resource restores the gem permissions to a minimum of
+# 755 on directories and 644 on ordinary files.
+#
+execute 'correct-chef-gem-permissions' do
+  gem_dir = Shellwords.escape(Gem.dir)
+  command "find #{gem_dir} -type f -exec chmod a+r {} \\; && " +
+          "find #{gem_dir} -type d -exec chmod a+rx {} \\;"
+  user 'root'
   action :nothing
 end.run_action(:run)

cookbooks/bcpc-hadoop/attributes/disks.rb

@@ -1,11 +1,29 @@
-# enumerate available disks and how they will be used
-avail_disks = node[:block_device].keys.select{ |d| d =~ /sd[b-z]/ }
-default[:bcpc][:hadoop][:disks] = {
-  :available_disks => avail_disks,
-  # keep at least that many disks for the :disk_reserve_roles
-  :role_min_disk => 2,
-  # we are reserving disks for the following
-  :reservation_requests => ["graphite_disk"],
-  # reservations will be saved here
-  :disk_reserve_roles => ["BCPC-Hadoop-Head"]
-} 
+# Enumerate available disks and how they will be used
+default[:bcpc][:hadoop][:disks].tap do |disks|
+  #
+  # What disks will bcpc and bcpc-hadoop feel free to blank?
+  # By default, all sd* devices, excluding sda.
+  #
+  # On our EFI-based VM builds, it's very important to omit sdb, as
+  # that is the 32 MB image containing iPXE.  (It's relatively
+  # harmless to overwrite it, but it will cause graphite to fail when
+  # /disk/0 fills up.)
+  #
+  # We have to check DMI because Ohai doesn't understand VBox EFI.
+  #
+  disks[:available_disks] =
+    if node[:dmi][:system][:product_name] == 'VirtualBox'
+      node[:block_device].keys.select{ |d| d =~ /sd[a-i]?[b-z]/ } - ['sdb']
+    else
+      node[:block_device].keys.select{ |d| d =~ /sd[a-i]?[b-z]/ }
+    end
+
+  # Keep at least this many disks for the :disk_reserve_roles
+  disks[:role_min_disk] = 2
+
+  # We are reserving disks for the following
+  disks[:reservation_requests] = ['graphite_disk']
+
+  # Reservations will be saved here
+  disks[:disk_reserve_roles] = ['BCPC-Hadoop-Head']
+end

cookbooks/bcpc-hadoop/libraries/utils.rb

@@ -218,40 +218,6 @@ def set_hosts
   node.default[:bcpc][:hadoop][:mysql_hosts] = get_node_attributes(HOSTNAME_ATTR_SRCH_KEYS,"mysql","bcpc")
 end
 
-#
-# Library function to get attributes for nodes that executes a particular recipe
-#
-def get_node_attributes(srch_keys,recipe,cookbook=cookbook_name)
-  node_objects = get_nodes_for(recipe,cookbook)
-  ret = get_req_node_attributes(node_objects,srch_keys)
-  return ret
-end
-
-#
-# Library function to retrieve required attributes from a array of node objects passed
-# Takes in an array of node objects and a search hash. Refer to comments for the constant
-# DEFAULT_NODE_ATTR_SRCH_KEYS regarding the format of the hash
-# returns a array of hash with the requested attributes
-# [ { :node_number => "val", :hostname => "nameval" }, ...]
-#
-def get_req_node_attributes(node_objects,srch_keys)
-  result = Array.new
-  node_objects.each do |obj|
-    temp = Hash.new
-    srch_keys.each do |name, key|
-      begin
-        val = key.split('.').reduce(obj) {|memo, key| memo[key]}
-      rescue NoMethodError
-        Chef::Log.fatal "Node #{obj} does not have key #{key}!"
-        raise
-      end
-      temp[name] = val
-    end
-    result.push(temp)
-  end
-  return result
-end
-
 #
 # Restarting of hadoop processes need to be controlled in a way that all the nodes
 # are not down at the sametime, the consequence of which will impact users. In order

cookbooks/bcpc-hadoop/metadata.rb

@@ -7,6 +7,7 @@
 version          '0.1.0'
 
 depends 'bcpc', '= 0.1.0'
+depends 'database'
 depends 'java', '>= 1.28.0'
 depends 'maven', '~> 2.1.1'
 depends 'pam'

cookbooks/bcpc-hadoop/recipes/graphite_to_zabbix.rb

@@ -49,7 +49,7 @@
     zbx = ZabbixApi.connect(
       :url => "https://#{node['bcpc']['management']['vip']}" +
         ":#{node['bcpc']['zabbix']['web_port']}/api_jsonrpc.php",
-      :user => 'admin',
+      :user => get_config!('zabbix-admin-user'),
       :password => "#{get_config!('password','zabbix-admin','os')}"
     )
     if zbx.nil?