Fix VMWare Tools by removing DKMS.

Apparently the last attempt at upgrading VMWare Tools didn't work.
Although DKMS was able to successfully compile the VMWare Tools kernel
modules, the resulting modules don't work correctly. This time we use
vanilla VMWare Tools without DKMS. We use VMWare Tools's own method of
recompiling modules on kernel upgrades.

Also fix more build robustness issues and VirtualBox issues while we're
at it.

Gemfile

@@ -1,2 +1,3 @@
 source 'https://www.rubygems.org'
 gem 'veewee', :git => 'https://github.com/phusion/veewee.git'
+gem 'nokogiri'

Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/phusion/veewee.git
-  revision: 3631547617a41bb6a071ff01c00f7cb633c1e629
+  revision: aee7b127cd9ab9f4469157357a1bbda54af3be2a
   specs:
     veewee (0.3.12)
       ansi (~> 1.3.0)
@@ -78,4 +78,5 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  nokogiri
   veewee!

LICENSE.txt

@@ -5,6 +5,8 @@ contain proprietary components (as is the case with the VMWare Fusion images,
 which include the VMWare Tools). Each component is copyrighted by their
 respective authors.
 
+The VMWare Tools' license can be found at https://www.vmware.com/download/eula/
+
 VMWare Fusion is a registered trademark of VMWare Inc.
 VMWare Tools is probably also a trademark (not sure whether registered) of
 VMWare Inc.

README.md

@@ -71,8 +71,8 @@ These Vagrant boxes are provided to you by [Phusion](http://www.phusion.nl/). Yo
 
 VirtualBox:
 
-    bundle exec rake virtualbox:build virtualbox:import
+    bundle exec rake virtualbox:all
 
 VMWare Fusion:
 
-    bundle exec rake vmware_fusion:build vmware_fusion:import
+    bundle exec rake vmware_fusion:all

Rakefile

@@ -1,4 +1,3 @@
-BOXES = ['ubuntu-12.04.3-amd64-vbox.box', 'ubuntu-12.04.3-amd64-vmwarefusion.box']
 VMWARE_TOOLS_URL = "http://softwareupdate.vmware.com/cds/vmw-desktop/fusion/6.0.2/1398658/packages/com.vmware.fusion.tools.linux.zip.tar"
 VMWARE_TOOLS_ARCHIVE = "VMwareTools-9.6.1-1378637.tar.gz"
 WEBSERVER = "juvia-helper.phusion.nl"
@@ -7,26 +6,81 @@ WEBROOT = "/srv/oss_binaries_passenger/vagrant/boxes"
 
 #### Boxes ####
 
-desc "Build VirtualBox box file"
-task "virtualbox:build" do
+desc "Build VirtualBox box file & import it into Vagrant"
+task "virtualbox:all" => ["virtualbox:build_image", "virtualbox:build_box", "virtualbox:import_box"]
+
+desc "Build VirtualBox image"
+task "virtualbox:build_image" do
 	sh "bundle exec veewee vbox build ubuntu-12.04.3-amd64-vbox --force --auto"
+	sh "bundle exec veewee vbox ssh ubuntu-12.04.3-amd64-vbox 'sudo poweroff'"
+	puts "Sleeping a few seconds, waiting for the VM to power off."
+	sh "sleep 30"
+end
+
+desc "Build VirtualBox box file"
+task "virtualbox:build_box" do
+	require 'nokogiri'
 	sh "bundle exec veewee vbox export ubuntu-12.04.3-amd64-vbox --force"
+	sh "rm -rf tmp && mkdir tmp && cd tmp && tar xf ../ubuntu-12.04-3-amd64-vbox.box"
+	doc = Nokogiri.XML(File.open("tmp/box.ovf", "r"))
+	# Remove DVD device which could cause problems for older VirtualBoxes:
+	# https://github.com/phusion/open-vagrant-boxes/issues/1
+	(doc / "StorageControllers > StorageController[name='IDE Controller'] > AttachedDevice[port='1']").remove
+	# Remove all Shared Folders created by Veewee. Fixes some warnings.
+	(doc / "SharedFolder").remove
+	File.open("tmp/box.ovf", "w") do |f|
+		doc.write_xml_to(f)
+	end
+	sh "cd tmp && tar -cf ../ubuntu-12.04-3-amd64-vbox.box *"
+	sh "rm -rf tmp"
 end
 
 desc "Import VirtualBox box file into Vagrant"
-task "virtualbox:import" do
+task "virtualbox:import_box" do
 	sh "vagrant box add phusion-open-ubuntu-12.04-amd64 ubuntu-12.04.3-amd64-vbox.box --force"
 end
 
 
-desc "Build VMWare Fusion box file"
-task "vmware_fusion:build" => "iso/_latest_vmware_tools.tar.gz" do
+desc "Build VMWare Fusion box file & import it into Vagrant"
+task "vmware_fusion:all" => ["vmware_fusion:build_image", "vmware_fusion:fixup_image",
+	"vmware_fusion:build_box", "vmware_fusion:import_box"]
+
+desc "Build VMWare Fusion image"
+task "vmware_fusion:build_image" => "iso/_latest_vmware_tools.tar.gz" do
 	sh "bundle exec veewee fusion build ubuntu-12.04.3-amd64-vmwarefusion --force --auto"
+	sh "bundle exec veewee fusion ssh ubuntu-12.04.3-amd64-vmwarefusion 'sudo poweroff'"
+	puts "Sleeping a few seconds, waiting for the VM to power off."
+	sh "sleep 30"
+end
+
+desc "Fix up VMWare Tools inside the VM"
+task "vmware_fusion:fixup_image" do
+	# After building the box, the kernel has been upgraded. We have to boot it
+	# in the new kernel at least once so that the VMWare Tools are properly compiled
+	# for the new kernel.
+	require 'shellwords'
+	sh "bundle exec veewee fusion up ubuntu-12.04.3-amd64-vmwarefusion"
+	sh "sleep 10"
+	sh "chmod 600 vagrant_insecure.key"
+	ssh_command = %Q{
+		set -e
+		while ! sudo modprobe vmhgfs; do
+			echo "`date`: VMWare Tools not yet compiled. Waiting..." 
+			sleep 5
+		done
+		echo "`date`: VMWare Tools are now compiled. Powering off VM."
+		sudo poweroff
+	}
+	sh "bundle exec veewee fusion ssh ubuntu-12.04.3-amd64-vmwarefusion #{Shellwords.escape ssh_command}"
+end
+
+desc "Build VMWare Fusion box file"
+task "vmware_fusion:build_box" do
 	sh "bundle exec veewee fusion export ubuntu-12.04.3-amd64-vmwarefusion --force"
 end
 
 desc "Import VMWare Fusion box file into Vagrant"
-task "vmware_fusion:import" do
+task "vmware_fusion:import_box" do
 	sh "vagrant box add phusion-open-ubuntu-12.04-amd64 ubuntu-12.04.3-amd64-vmwarefusion.box --force"
 end
 
@@ -54,12 +108,18 @@ end
 
 #### Release #####
 
-desc "Upload boxes to a public server"
-task :upload => BOXES do
-	BOXES.each do |box|
+def create_release_task(name, box_file)
+	desc "Release #{name} box file to a public server"
+	task "release:#{name}" => box_file do
 		sh "ssh", WEBSERVER, "rm -rf #{WEBROOT}/tmp && mkdir #{WEBROOT}/tmp"
-		sh "scp", box, "#{WEBSERVER}:#{WEBROOT}/tmp/"
-		sh "md5sum #{box} | ssh #{WEBSERVER} tee #{WEBROOT}/tmp/#{box}.md5.txt"
+		sh "scp #{box_file} #{WEBSERVER}:#{WEBROOT}/tmp/"
+		sh "md5sum #{box_file} | ssh #{WEBSERVER} tee #{WEBROOT}/tmp/#{box_file}.md5.txt"
 		sh "ssh", WEBSERVER, "mv #{WEBROOT}/tmp/* #{WEBROOT}/ && rm -rf #{WEBROOT}/tmp"
 	end
 end
+
+desc "Release all box files to a public server"
+task "release" => ["release:virtualbox", "release:vmware_fusion"]
+
+create_release_task("virtualbox", "ubuntu-12.04.3-amd64-vbox.box")
+create_release_task("vmware_fusion", "ubuntu-12.04.3-amd64-vmwarefusion.box")

definitions/ubuntu-12.04.3-amd64-vbox/cleanup.sh

@@ -1,7 +1,9 @@
 # Remove items used for building, since they aren't needed anymore
 set -x
 
-apt-get -y remove build-essential
+# We don't remove build-essential. It is required for recompiling the guest
+# additions after upgrading the kernel.
+
 apt-get -y autoremove
 apt-get clean
 

definitions/ubuntu-12.04.3-amd64-vbox/definition.rb

@@ -25,7 +25,7 @@
   :ssh_login_timeout => "10000",
   :ssh_user => "vagrant",
   :ssh_password => "vagrant",
-  :ssh_key => "",
+  :ssh_key => "../../vagrant_insecure.key",
   :ssh_host_port => "7222",
   :ssh_guest_port => "22",
   :sudo_cmd => "echo '%p'|sudo -S bash '%f'",

definitions/ubuntu-12.04.3-amd64-vbox/vmfusion.sh

@@ -12,8 +12,7 @@ set +x
 ###################
 
 #!/bin/bash
-# Automatic install and configure VMware Tools using DKMS 
-# dkms.conf is an modified version of open-vm-tools' dkms.conf
+# Automatic install and configure VMware Tools
 #
 # Idea for this script is from http://www.l4l.be/docs/virt/openvmtools_ubuntu810.php
 #
@@ -163,62 +162,6 @@ else
   done
 fi
 
-echo -n "Generating /usr/src/vmware-tools-$VERSION/dkms.conf: "
-cat > /usr/src/vmware-tools-$VERSION/dkms.conf <<EOF
-PACKAGE_NAME=vmware-tools
-PACKAGE_VERSION=$VERSION
-MAKE_CMD_TMPL="make VM_UNAME=\$kernelver \
-               MODULEBUILDDIR=\$dkms_tree/\$PACKAGE_NAME/\$PACKAGE_VERSION/build"
-
-MAKE[0]="\$MAKE_CMD_TMPL -C vmblock-only VM_UNAME=\$kernelver &&   \\
-         \$MAKE_CMD_TMPL -C vmci-only VM_UNAME=\$kernelver &&      \\
-         \$MAKE_CMD_TMPL -C vmhgfs-only VM_UNAME=\$kernelver &&    \\
-         \$MAKE_CMD_TMPL -C vmmemctl-only VM_UNAME=\$kernelver &&  \\
-         \$MAKE_CMD_TMPL -C vmxnet-only VM_UNAME=\$kernelver &&    \\
-         \$MAKE_CMD_TMPL -C vsock-only VM_UNAME=\$kernelver"
-BUILT_MODULE_NAME[0]="vmblock"
-BUILT_MODULE_NAME[1]="vmci"
-BUILT_MODULE_NAME[2]="vmhgfs"
-BUILT_MODULE_NAME[3]="vmmemctl"
-BUILT_MODULE_NAME[4]="vmxnet"
-BUILT_MODULE_NAME[5]="vsock"
-BUILT_MODULE_LOCATION[0]="vmblock-only/"
-BUILT_MODULE_LOCATION[1]="vmci-only/"
-BUILT_MODULE_LOCATION[2]="vmhgfs-only/"
-BUILT_MODULE_LOCATION[3]="vmmemctl-only/"
-BUILT_MODULE_LOCATION[4]="vmxnet-only/"
-BUILT_MODULE_LOCATION[5]="vsock-only/"
-DEST_MODULE_LOCATION[0]="/kernel/fs/vmblock"
-DEST_MODULE_LOCATION[1]="/kernel/drivers/misc"
-DEST_MODULE_LOCATION[2]="/kernel/fs/vmhgfs"
-DEST_MODULE_LOCATION[3]="/kernel/drivers/misc"
-DEST_MODULE_LOCATION[4]="/kernel/drivers/net"
-DEST_MODULE_LOCATION[5]="/kernel/drivers/misc"
-AUTOINSTALL="YES"
-EOF
-echo "Ok"
-
-if ! dpkg -L dkms >/dev/null 2>&1; then
-  echo "Installing dkms: " 
-  if apt-get -qq -y install dkms >/dev/null 2>&1; then
-    echo "Ok"
-  else
-    echo "Error"
-    exit 7
-  fi
-else 
-  if dkms status | grep -q "^vmware-tools" ; then 
-    echo -n "Removing old dkms definitions: "
-    for VRSN in $(dkms status | awk '/^vmware-tools/{ver=$2; gsub(/[,:]/, "",ver); printf "%s\n", ver;}'); 
-    do
-      dkms remove -m vmware-tools -v $VRSN --all >/dev/null 2>&1 && echo -n "$VRSN "
-      if [ -d /usr/src/vmware-tools-$VRSN -a "$VRSN" != "$VERSION" ]; then
-        rm -rf /usr/src/vmware-tools-$VRSN >/dev/null 2>&1
-      fi
-    done
-    echo
-  fi
-fi
 
 DISTRIB=$(uname -r | cut -d- -f3)
 if ! dpkg -L linux-headers-$DISTRIB >/dev/null 2>&1; then
@@ -251,58 +194,38 @@ if ! dpkg -L build-essential >/dev/null 2>&1; then
   fi
 fi
 
-echo -n "Adding vmware-tools to dkms: "
-if dkms add -m vmware-tools -v $VERSION >/dev/null 2>&1; then
-  echo "Ok"
-else
-  echo "Error"
-  exit 7
-fi
-echo -n "Building kernel modules for $(uname -r): "
-if dkms build -m vmware-tools -v $VERSION -k $(uname -r) >/dev/null 2>&1; then
-  echo "Ok"
-else
-  echo "Error"
-  exit 7
-fi
-echo -n "Installing kernel modules for $(uname -r): "
-if dkms install --force -m vmware-tools -v $VERSION -k $(uname -r) >/dev/null 2>&1; then
-  echo "Ok"
-else
-  echo "Error"
-  exit 7
-fi
-
-echo -n "Installing vmware-tools: "
-if perl /tmp/vmware-tools-distrib/vmware-install.pl -d >/dev/null 2>&1; then
-  echo "Ok"
-else
-  echo "Error"
-  exit 7
-fi
 
 echo -n "Patching vmware-tools: "
+set +e
 (
-  set +e
-  cd /etc/vmware-tools
-  # Allow vmware-tools to load kernel modules compiled by DKMS.
+  set -e
+  cd /tmp/vmware-tools-distrib
+  # Enable automatic recompiling of vmware-tools kernel modules on kernel upgrades.
   patch -N -p0 >/dev/null 2>&1 <<"EOF"
---- services.sh.orig    2013-07-12 15:21:22.734631410 +0000
-+++ services.sh 2013-07-12 15:31:04.786088356 +0000
-@@ -866,7 +866,8 @@
-    local module_path="`vmware_getModPath $1`"
-    local module_name="`vmware_getModName $1`"
-    /sbin/insmod -s -f "$module_path" >/dev/null 2>&1 || \
--       /sbin/insmod -s -f "$module_name" >/dev/null 2>&1 || exit 1
-+       /sbin/insmod -s -f "$module_name" >/dev/null 2>&1 || \
-+       /sbin/modprobe "$module_name" >/dev/null 2>&1 || exit 1
-    return 0
+--- bin/vmware-config-tools.pl.orig 2014-01-30 17:56:40.570846915 +0100
++++ bin/vmware-config-tools.pl  2014-01-30 17:56:51.980548609 +0100
+@@ -10837,7 +10837,7 @@
+ EOF
+
+    $ans = get_persistent_answer($msg, 'AUTO_KMODS_ENABLED_ANSWER', 'yesno',
+-                                'no');
++                                'yes');
+    db_add_answer('AUTO_KMODS_ENABLED', $ans);
  }
 
 EOF
-  sed -i 's/^answer VMHGFS_CONFED no$/answer VMHGFS_CONFED yes/' locations
 )
-if [[ "$?" = 0 ]]; then
+status=$?
+set -e
+if [[ "$status" = 0 ]]; then
+  echo "Ok"
+else
+  echo "Error"
+  exit 7
+fi
+
+echo -n "Installing vmware-tools: "
+if perl /tmp/vmware-tools-distrib/vmware-install.pl -d >/dev/null 2>&1; then
   echo "Ok"
 else
   echo "Error"

vagrant_insecure.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI
+w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP
+kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2
+hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO
+Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW
+yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd
+ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1
+Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf
+TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK
+iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A
+sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf
+4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP
+cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk
+EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN
+CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX
+3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG
+YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj
+3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+
+dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz
+6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC
+P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF
+llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ
+kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH
++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ
+NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s=
+-----END RSA PRIVATE KEY-----