Add proper ping/status checks before volume creation.

This adds proper (optional) ping checks with fping and gluster peer
status checks to ensure the peer are available before a volume create
command. This required rewriting of the xml.py hack which helps puppet
interface with the xml formatted gluster cli output. In addition,
downstream commands such as volume::property gained checks to ensure the
volume was present beforehand. While it is not obvious, it should be
noted that because of the distributed nature of glusterfs, more than one
puppet run will be required for complete deployment. With these patches,
individual runs shouldn't ever end in temporary error as they used too.

files/xml.py

@@ -1,6 +1,5 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2012+ Jordi Gutiérrez Hermoso <jordigh@octave.org>
 # Copyright (C) 2012-2013+ James Shubin
 # Written by James Shubin <james@shubin.ca>
 #
@@ -17,24 +16,69 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# thanks to Jordi for fighting with the xml for me so that I didn't have to :)
+# 	EXAMPLE:
+#	$ gluster peer status --xml | ./xml.py --connected <PEER1> <PEER2> <PEERn>
+#	<BOOL>
 
 # 	EXAMPLE:
-#	$ gluster volume --xml info <VOLNAME> | ./xml.py <KEY>
+#	$ gluster volume --xml info <VOLNAME> | ./xml.py --property <KEY>
 #	<VALUE>
 
 import sys
 import lxml.etree as etree
 
-if len(sys.argv) != 2:
-	sys.exit(1)
+argv = sys.argv
+argv.pop(0)	# get rid of $0
+
+if len(argv) < 1:
+	sys.exit(3)
+
+mode = argv.pop(0)
+tree = etree.parse(sys.stdin)
+root = tree.getroot()
+
+# are all the hostnames in argv connected ?
+if mode == '--connected':
+	store = {}
+	peers = [x for x in argv if x != '']
+
+	for i in root.findall('.//peerStatus'):
+		p = i.find('peer')
+		h = p.find('hostname').text
+		c = (str(p.find('connected').text) == '1')	# connected
+		store[h] = c	# save for later...
+
+	# if no peers specified, assume we should check all...
+	if len(peers) == 0:
+		peers = store.keys()
 
-t = etree.parse(sys.stdin)
-r = t.getroot()
-v = [x.find('value').text for x in r.findall('.//option') if x.find('name').text == str(sys.argv[1])]
-if len(v) == 1:
-	print v[0]
+	for i in peers:
+		if i in store.keys():
+			if not store[i]:
+				# someone is unconnected
+				sys.exit(1)
+		else:
+			# we're looking for a peer that isn't peered yet
+			sys.exit(2)
+
+	# must be good!
 	sys.exit(0)
-else:			# more than one value found
-	sys.exit(1)
+
+elif mode == '--property':
+	if len(argv) != 1:
+		sys.exit(3)
+
+	store = []
+	for i in root.findall('.//option'):
+		if i.find('name').text == str(argv[0]):
+			store.append(i.find('value').text)
+
+	if len(store) == 1:
+		print(store[0])
+		sys.exit(0)
+	else:			# more than one value found
+		sys.exit(1)
+
+# else:
+sys.exit(3)
 

manifests/volume.pp

@@ -21,8 +21,18 @@
 	$replica = 1,
 	$stripe = 1,
 	$vip = '',		# vip of the cluster (optional but recommended)
+	$ping = true,		# do we want to include fping checks ?
 	$start = undef		# start volume ? true, false (stop it) or undef
 ) {
+	include gluster::xml
+	include gluster::vardir
+	if $ping {
+		include gluster::volume::ping
+	}
+
+	#$vardir = $::gluster::vardir::module_vardir	# with trailing slash
+	$vardir = regsubst($::gluster::vardir::module_vardir, '\/$', '')
+
 	# TODO: if using rdma, maybe we should pull in the rdma package... ?
 	$valid_transport = $transport ? {
 		'rdma' => 'rdma',
@@ -61,6 +71,32 @@
 	# add /${name} to the end of each: brick:/path entry
 	$brick_spec = inline_template("<%= bricks.collect {|x| ''+x.chomp('/')+'/${name}' }.join(' ') %>")
 
+	# get the list of bricks fqdn's that don't have our fqdn
+	$others = inline_template("<%= bricks.find_all{|x| x.split(':')[0] != '${fqdn}' }.collect {|y| y.split(':')[0] }.join(' ') %>")
+
+	$fping = sprintf("/usr/sbin/fping -q %s", $others)
+	$status = sprintf("/usr/sbin/gluster peer status --xml | ${vardir}/xml.py --connected %s", $others)
+
+	$onlyif = $ping ? {
+		false => "${status}",
+		default => [
+			"${fping}",
+			"${status}",
+		],
+	}
+
+	$require = $ping ? {
+		false => [
+			Service['glusterd'],
+			File["${vardir}/xml.py"],	# status check
+		],
+		default => [
+			Service['glusterd'],
+			Package['fping'],
+			File["${vardir}/xml.py"],	# status check
+		],
+	}
+
 	# run if vip not defined (bypass mode) or if vip exists on this machine
 	if ($vip == '' or $vipif != '') {
 		# NOTE: This should only happen on one host!
@@ -71,13 +107,15 @@
 		# vip) or one (the vip node, when using vip) before it succeeds
 		# because it shouldn't work until all the bricks are available,
 		# which per node will happen right before this runs.
+		# fping all the other nodes to ensure they're up for creation
 		# EXAMPLE: gluster volume create test replica 2 transport tcp annex1.example.com:/storage1a/test annex2.example.com:/storage2a/test annex3.example.com:/storage3b/test annex4.example.com:/storage4b/test annex1.example.com:/storage1c/test annex2.example.com:/storage2c/test annex3.example.com:/storage3d/test annex4.example.com:/storage4d/test
 		exec { "/usr/sbin/gluster volume create ${name} ${valid_replica}${valid_stripe}transport ${valid_transport} ${brick_spec}":
 			logoutput => on_failure,
 			unless => "/usr/sbin/gluster volume list | /bin/grep -qxF '${name}' -",	# add volume if it doesn't exist
+			onlyif => $onlyif,
 			#before => TODO?,
 			#require => Gluster::Brick[$bricks],
-			require => Service['glusterd'],
+			require => $require,
 			alias => "gluster-volume-create-${name}",
 		}
 	}
@@ -99,6 +137,7 @@
 			# try to start volume if stopped
 			exec { "/usr/sbin/gluster volume start ${name}":
 				logoutput => on_failure,
+				onlyif => "/usr/sbin/gluster volume list | /bin/grep -qxF '${name}' -",
 				unless => "/usr/sbin/gluster volume status ${name}",	# returns false if stopped
 				require => Exec["gluster-volume-create-${name}"],
 				alias => "gluster-volume-start-${name}",

manifests/volume/ping.pp

@@ -0,0 +1,25 @@
+# Simple? gluster module by James
+# Copyright (C) 2012-2013+ James Shubin
+# Written by James Shubin <james@shubin.ca>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+class gluster::volume::ping {
+
+	package { 'fping':		# for checking other bricks are up
+		ensure => present,
+	}
+}
+
+# vim: ts=8

manifests/volume/property.pp

@@ -21,7 +21,7 @@
 	$value,
 	$autotype = true		# set to false to disable autotyping
 ) {
-	include gluster::volume::property::base
+	include gluster::xml
 	include gluster::vardir
 	#$vardir = $::gluster::vardir::module_vardir	# with trailing slash
 	$vardir = regsubst($::gluster::vardir::module_vardir, '\/$', '')
@@ -82,7 +82,8 @@
 	# FIXME: check that the value we're setting isn't the default
 	# FIXME: you can check defaults with... gluster volume set help | ...
 	exec { "/usr/sbin/gluster volume set ${volume} ${key} ${safe_value}":
-		unless => "/usr/bin/test \"`/usr/sbin/gluster volume --xml info ${volume} | ${vardir}/xml.py ${key}`\" = '${safe_value}'",
+		unless => "/usr/bin/test \"`/usr/sbin/gluster volume --xml info ${volume} | ${vardir}/xml.py --property ${key}`\" = '${safe_value}'",
+		onlyif => "/usr/sbin/gluster volume list | /bin/grep -qxF '${volume}' -",
 		logoutput => on_failure,
 		require => [
 			Gluster::Volume[$volume],

manifests/volume/property/base.pp → manifests/xml.pp

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-class gluster::volume::property::base {
+class gluster::xml {
 	include gluster::vardir
 
 	package { 'python-lxml':		# for parsing gluster xml output