[#9] - Installing Zeppelin to the Sandbox or Cluster (#11)

* [Zeppelin Ambari Service] - Adding submodule from hw gallery

I decided to use a git submodule to house the amber service so that
it’s possible to update and I also don’t have to create the service
from scratch

* adding hep-select installer

this is required by the ambari services submodules that we’ll be using
so it’s necessary that it is installed

* better code coverage for hdp-select installer

* fixing output collection for real

Didn’t use the array out output for stderr and stdout before

* Updates to curl_client for service installer

Needed to be able to add more arguments in

* Adding Zeppelin installer function

* Last bits for coverage

Resolves #9

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "ambari-services/ambari-zeppelin-service"]
+	path = ambari-services/ambari-zeppelin-service
+	url = https://github.com/hortonworks-gallery/ambari-zeppelin-service

conf/global-config.conf

@@ -0,0 +1,16 @@
+[DEMO]
+name="Solutions Demo"
+
+[AMBARI]
+server=localhost
+port=8080
+username=admin
+password=admin
+cluster_name=Sandbox
+proto=http
+
+[ZEPPELIN]
+notebooks_directory="/var/zeppelin/notebooks"
+
+[NIFI]
+INSTALL_DIR="/opt"

conf/service-installer.conf

@@ -0,0 +1,15 @@
+[SERVICES]
+service-names=["ZEPPELIN"]
+
+
+[ZEPPELIN]
+install-commands=[ "hdp-select status hadoop-client | sed 's/hadoop-client - \\([0-9]\\.[0-9]\\).*/\\1/'", "cp -r ../ambari-services/ambari-zeppelin-service /var/lib/ambari-server/resources/stacks/HDP/$VERSION/services/ZEPPELIN", "ambari-server restart"]
+server=localhost
+port=9995
+
+# Repo Links for HDP-SELECT
+[HDP-SELECT]
+ubuntu12="http://public-repo-1.hortonworks.com/HDP/ubuntu12/2.x/updates/2.4.2.0/pool/main/h/hdp-select/hdp-select_2.4.2.0-258_all.deb"
+ubuntu14="http://public-repo-1.hortonworks.com/HDP/ubuntu14/2.x/updates/2.4.2.0/pool/main/h/hdp-select/hdp-select_2.4.2.0-258_all.deb"
+centos6="http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.4.0.0/hdp-select/hdp-select-2.4.0.0-169.el6.noarch.rpm"
+centos7="http://public-repo-1.hortonworks.com/HDP/centos7/2.x/updates/2.4.0.0/hdp-select/hdp-select-2.4.0.0-169.el6.noarch.rpm"

scripts/curl_client.py

@@ -28,10 +28,14 @@ def set_server(self, server):
 
 	# A number between 0 and 65535 
 	def set_port(self, port):
-		if not type(port) is int:
+		int_port = -1
+		try:
+			int_port = int(port)
+		except ValueError as e:
 			raise ValueError('Server port was not of type: int')
-		if port > 0 and port <= 65535:
-			self.port = port
+
+		if int_port > 0 and int_port <= 65535:
+			self.port = int_port
 		else:
 			raise ValueError('Server port must be between 0 and 65535. Value was ' + str(port))
 
@@ -42,7 +46,7 @@ def set_port(self, port):
 	# A list of query parameters
 	# 	['param1=value1', 'param2=value2']
 
-	def make_request(self, verb, request, query=''):
+	def make_request(self, verb, request, options='', query=''):
 
 		if not (verb == 'GET' or verb == 'POST' or verb == 'PUT' or verb == 'DELETE'):
 			raise ValueError('HTTP Verb must be one of GET|PUT|POST|DELETE')
@@ -58,7 +62,7 @@ def make_request(self, verb, request, query=''):
 
 		method = '-X ' + verb
 
-		call = ' '.join(['curl -sS', credentials, method, url])
+		call = ' '.join(['curl -sS', credentials, method, options, url])
 		output = self.cmd.run(call)
 		return output
 

scripts/service_installer.py

@@ -0,0 +1,160 @@
+# Script which installs Zeppelin as an Ambari Service
+import config, sys, platform, json, time
+from shell import Shell
+from curl_client import CurlClient
+
+def install_hdp_select():
+	dist_info = platform.linux_distribution()
+	if(len(dist_info[0]) == 0):
+		raise EnvironmentError('You must be running a linux distribution to install hdp-select')
+
+#	Only want to get distro name
+#	Determine first distro name
+#		Then determine the version (first char char for centos )
+	distro = dist_info[0].lower()
+	fullname = distro
+	ver = dist_info[1]
+	if 'centos' in distro: # Get First 1/2 nums in version string
+		fullname = fullname + dist_info[1][0]
+	elif 'ubuntu' in distro:
+		if (len(dist_info[1]) < 2):
+			fullname = fullname + dist_info[1][0]
+		else:
+			fullname = fullname + dist_info[1][0] + dist_info[1][1]
+
+	conf = config.read_config('../conf/service-installer.conf')
+	urls = conf['HDP-SELECT']
+	url = ''
+	if fullname == 'centos6':	
+		url = urls['centos6']
+	elif fullname == 'centos7':
+		url = urls['centos7']
+	elif fullname == 'ubuntu12':
+		url = urls['ubuntu12']
+	elif fullname == 'ubuntu14':
+		url = urls['ubuntu14']
+
+	res = ''
+
+	if len(url) == 0:
+		raise EnvironmentError('Must be using one of: CentOS 6.x, CentOS 7.x, Ubuntu 12.x, Ubuntu 14.x')
+	elif 'centos' in fullname:
+		cmd = Shell()
+		output = cmd.run('yum install -y ' + url)
+		res = cmd.run('which hdp-select')
+	elif 'ubuntu' in fullname:
+		cmd = Shell()
+		output = cmd.run('wget ' + url + ' -O ./hdp-select.deb')
+		output = cmd.run('dpkg -i ' + './hdp-select.deb')
+		res = cmd.run('which hdp-select')
+
+	if len(res[0]) == 0:
+		return False
+	else:
+		return True
+
+def is_hdp_select_installed():
+	sh = Shell()
+	output = sh.run('which hdp-select')
+
+	if len(output[0]) == 0:
+		return False
+	else:
+		return True
+
+def is_ambari_installed():
+	sh = Shell()
+	output = sh.run('which ambari-server')
+	if len(output[0]) == 0:
+		return False
+	else:
+		return True
+
+
+
+def install_zeppelin(conf_dir):
+
+	if not conf_dir.endswith('/'):
+		conf_dir += '/'
+
+	if not is_ambari_installed():
+		raise EnvironmentError('You must install the demo on the same node as the Ambari server. Install Ambari here or move to another node with Ambari installed before continuing')
+
+
+	if not is_hdp_select_installed():
+		installed = install_hdp_select()
+		if not installed:
+			raise EnvironmentError('hdp-select could not be installed. Please install it manually and then re-run the setup.')
+
+	conf = config.read_config(conf_dir + 'service-installer.conf')
+	cmds = conf['ZEPPELIN']['install-commands']
+	cmds = json.loads(conf['ZEPPELIN']['install-commands'])
+
+	sh = Shell()
+#	print(sh.run('pwd')[0])
+	version = sh.run(cmds[0])
+#	print("HDP-VERSION: " + str(version[0]))
+	fixed_cmd = cmds[1].replace('$VERSION', str(version[0])).replace('\n', '')
+#	print('FIXED COPY COMMAND: ' + fixed_cmd)
+	copy = sh.run(fixed_cmd)
+#	print("COPY OUTPUT: " + copy[0])
+	restart = sh.run(cmds[2])
+#	print("Restart output: " + restart[0])
+
+
+	print("Please open the Ambari Interface and manually deploy the Zeppelin Service.")
+	raw_input("Press enter twice to continue...")
+	raw_input("Press enter once to continue...")
+
+#	 We've copied the necessary files. Once that completes we need to add it to Ambari
+
+	print('Checking to make sure service is installed')
+	ambari = config.read_config(conf_dir + 'global-config.conf')['AMBARI']
+	installed = check_ambari_service_installed('ZEPPELIN', ambari)
+	cont = ''
+	if not installed:
+		print('Unable to contact Ambari Server. Unsure whether or not Zeppelin was installed')
+		while not (cont == 'y' or cont == 'n'):
+			cont = raw_input('Continue attempt to set up Zeppelin for demo?(y/n)')
+			if not (cont == 'y' or cont == 'n'):
+				print('Please enter "y" or "n"')
+	else:
+		cont = 'y'
+
+	if cont == 'n':
+		return False
+	elif cont == 'y':
+		return True
+
+def check_ambari_service_installed(service_name, ambari_config):
+
+	curl = CurlClient(username=ambari_config['username'], password=ambari_config['password'], port=ambari_config['port'], server=ambari_config['server'], proto=ambari_config['proto'])
+
+	cluster_name = ambari_config['cluster_name']
+	request = '/api/v1/clusters/' + cluster_name + '/services/' + service_name
+	attempts = 0
+	while attempts < 10:
+		output = curl.make_request('GET', request, '-i')
+		if '200 OK' in output[0]:
+			print('Service Installed Sucessfully')
+			return True
+		else:
+			attempts += 1
+			raw_input('Could not connect.' + str(10-attempts) + ' remaining. Press any key to continue')
+
+	return False
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

tests/test_curlclient.py

@@ -10,9 +10,9 @@
 resErr = ['', '\'msg\':\'err\'']
 
 def mocked_request(*args, **kwargs):
-	if args[0] == 'curl -sS -u admin:admin -X GET http://demo-server:9090/api/v1/test?':
+	if '/api/v1/test' in args[0]:
 		return res1
-	elif args[0] == 'curl -sS -u admin:admin -X GET http://demo-server:9090/api/v1/bad?':
+	elif '/api/v1/bad' in args[0]:
 		return res2
 	else:
 		return res3