[#8] - POST Zeppelin Notebooks [PR #15]

* Updating config to auto search for the config directory

This helps remove many arguments from the service installer functions

* Adding POSTing feature for Zeppelin Notebook

Now possible to POST notebooks to a running Zeppelin install

* Removed a test accidentally on previous merge.

conf/service-installer.conf

@@ -4,6 +4,7 @@ service-names=["ZEPPELIN", "NIFI"]
 
 [ZEPPELIN]
 install-commands=[ "hdp-select status hadoop-client | sed 's/hadoop-client - \\([0-9]\\.[0-9]\\).*/\\1/'", "cp -r ../ambari-services/ambari-zeppelin-service /var/lib/ambari-server/resources/stacks/HDP/$VERSION/services/ZEPPELIN", "ambari-server restart"]
+protocol=http
 server=localhost
 port=9995
 

scripts/config.py

@@ -12,8 +12,7 @@
 # All params are read as strings
 
 def read_config(configFile):
-	cwd = os.getcwd()
-	path = cwd + '/' + configFile
+	path = get_conf_dir() + configFile
 
 	if not os.path.isfile(path):
 		raise IOError('could not find file at '+path )
@@ -27,4 +26,19 @@ def read_config(configFile):
 		for key in config.options(section):
 			params[section][key] = config.get(section, key)
 
-	return params
+	return params
+
+def get_conf_dir():
+	dirs = [str(os.getcwd()), str(os.curdir), '../']
+
+	for loc in dirs:
+		if not (str(loc).endswith('/')):
+			loc += '/'
+		loc += 'conf/'
+		if(os.path.exists(loc)):
+			return loc
+
+	raise EnvironmentError('Could not find conf directory')
+
+
+

scripts/service_installer.py

@@ -1,5 +1,5 @@
 # Script which installs Zeppelin as an Ambari Service
-import config, sys, platform, json, time
+import config, sys, platform, json, time, os
 from shell import Shell
 from curl_client import CurlClient
 
@@ -22,7 +22,7 @@ def install_hdp_select():
 		else:
 			fullname = fullname + dist_info[1][0] + dist_info[1][1]
 
-	conf = config.read_config('../conf/service-installer.conf')
+	conf = config.read_config('service-installer.conf')
 	urls = conf['HDP-SELECT']
 	url = ''
 	if fullname == 'centos6':	
@@ -52,7 +52,7 @@ def install_hdp_select():
 		return False
 	else:
 		return True
-
+	
 def is_hdp_select_installed():
 	sh = Shell()
 	output = sh.run('which hdp-select')
@@ -70,12 +70,35 @@ def is_ambari_installed():
 	else:
 		return True
 
+# Uses the conf/zeppelin/notes directory to upload pre-made notebooks
+def add_zeppelin_notebooks():
+	all_success = True
+	note_dir = config.get_conf_dir() + 'zeppelin/notes'
+	for item in os.listdir(note_dir):
+		item_path = note_dir + '/' + item
+		if os.path.isfile(item_path) and str(item).endswith('.json'):
+			result = post_notebook(item_path)
+			if not result:
+				all_success = False
+	return all_success
+
+#			log ("POSTED NOTEBOOK: " + str(post_notebook(item_path)))
+
+
+def post_notebook(notebook_path):
+	conf = config.read_config('service-installer.conf')['ZEPPELIN']
+	client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port']))
+	path = '/api/notebook'
+
+	output = client.make_request('POST', path, options='-i -H "Content-Type: application/json" -d @' + notebook_path )
+	if '201 created' in output[0].lower():
+#		log successful note created
+		return True
+	else:
+#		log failed note creation (and to import manually)
+		return False
 
-def install_zeppelin(conf_dir):
-
-	if not conf_dir.endswith('/'):
-		conf_dir += '/'
-
+def install_zeppelin():
 	if not is_ambari_installed():
 		raise EnvironmentError('You must install the demo on the same node as the Ambari server. Install Ambari here or move to another node with Ambari installed before continuing')
 
@@ -85,7 +108,7 @@ def install_zeppelin(conf_dir):
 		if not installed:
 			raise EnvironmentError('hdp-select could not be installed. Please install it manually and then re-run the setup.')
 
-	conf = config.read_config(conf_dir + 'service-installer.conf')
+	conf = config.read_config('service-installer.conf')
 	cmds = conf['ZEPPELIN']['install-commands']
 	cmds = json.loads(conf['ZEPPELIN']['install-commands'])
 
@@ -108,7 +131,7 @@ def install_zeppelin(conf_dir):
 #	 We've copied the necessary files. Once that completes we need to add it to Ambari
 
 	print('Checking to make sure service is installed')
-	ambari = config.read_config(conf_dir + 'global-config.conf')['AMBARI']
+	ambari = config.read_config('global-config.conf')['AMBARI']
 	installed = check_ambari_service_installed('ZEPPELIN', ambari)
 	cont = ''
 	if not installed:

tests/test_config.py

@@ -1,29 +1,53 @@
+import mock, unittest
 from env import scripts
 from scripts import config
 from ConfigParser import MissingSectionHeaderError
 
-def test_good_file():
-	params = config.read_config('res/good-test.properties')
-	assert params['SECTION1']['key1'] == 'val1'
-	assert params['SECTION2']['key2'] == 'val2'
-	assert params['SECTION3']['key3'] == 'val3'
-	assert params['SECTION3']['key4'] == 'val4'
-	assert len(params['SECTION3']) > 1
-	assert len(params['SECTION2']) > 0
-	assert len(params['SECTION1']) > 0
-	assert len(params) == 3
-
-def test_missing_header():
-	try:
-		params = config.read_config('res/bad-test.properties')
-		assert 0
-	except MissingSectionHeaderError as err:
-		assert 1
-
-def test_missing_file():
-	try:
-		params = config.read_config('nofile')
-		assert 0
-	except IOError as e:
-		if 'could not find file' not in e.message:
-			assert 0
+class TestConfig(unittest.TestCase):
+
+	@mock.patch('scripts.config.get_conf_dir', return_value='')
+	def test_good_file(self, mock1):
+		params = config.read_config('res/good-test.properties')
+		assert params['SECTION1']['key1'] == 'val1'
+		assert params['SECTION2']['key2'] == 'val2'
+		assert params['SECTION3']['key3'] == 'val3'
+		assert params['SECTION3']['key4'] == 'val4'
+		assert len(params['SECTION3']) > 1
+		assert len(params['SECTION2']) > 0
+		assert len(params['SECTION1']) > 0
+		assert len(params) == 3
+
+	@mock.patch('scripts.config.get_conf_dir', return_value='')
+	def test_missing_header(self, mock1):
+		try:
+			params = config.read_config('res/bad-test.properties')
+			assert 0
+		except MissingSectionHeaderError as err:
+			assert 1
+
+	@mock.patch('scripts.config.get_conf_dir', return_value='')
+	def test_missing_file(self, mock1):
+		try:
+			params = config.read_config('nofile')
+			assert 0
+		except IOError as e:
+			if 'could not find file' not in e.message:
+				assert 0
+
+	@mock.patch('os.path.exists', return_value=False)
+	def test_bad_env(self, mock1):
+		try:
+			config.get_conf_dir()
+			assert 0
+		except EnvironmentError as e:
+			assert str(e) == 'Could not find conf directory'
+
+
+	@mock.patch('os.path.exists', return_value=True)
+	@mock.patch('os.getcwd', return_value='adir')
+	def test_good_env(self, mock1, mock2):
+		try:
+			cdir = config.get_conf_dir()
+			assert 'conf/' in cdir
+		except EnvironmentError as e:
+			assert str(e) == 'Could not find conf directory'

tests/test_service_installer.py

@@ -106,7 +106,7 @@ class TestZeppelinInstall(unittest.TestCase):
 	@mock.patch('scripts.service_installer.is_ambari_installed', return_value=False)
 	def test_zeppelin_ambari_bad(self, mock):
 		try:
-			service_installer.install_zeppelin('../conf/service-installer.conf')
+			service_installer.install_zeppelin()
 			self.fail('Cannot continue installation without Ambari')
 		except EnvironmentError as e:
 			assert str(e.message) == 'You must install the demo on the same node as the Ambari server. Install Ambari here or move to another node with Ambari installed before continuing'
@@ -116,7 +116,7 @@ def test_zeppelin_ambari_bad(self, mock):
 	@mock.patch('scripts.service_installer.install_hdp_select', return_value=False)
 	def test_zeppelin_ambari_good(self, mock, mock2, mock3): #Also HDP select bad
 		try:
-			service_installer.install_zeppelin('../conf/service-installer.conf')
+			service_installer.install_zeppelin()
 			self.fail('Cannot continue installation without hdp-select')
 		except EnvironmentError as e:
 			assert str(e.message) == 'hdp-select could not be installed. Please install it manually and then re-run the setup.'
@@ -127,7 +127,7 @@ def test_zeppelin_ambari_good(self, mock, mock2, mock3): #Also HDP select bad
 	@mock.patch('scripts.service_installer.check_ambari_service_installed', return_value=True)
 	@mock.patch('__builtin__.raw_input', return_value='y')
 	def test_zeppelin_check_is_good(self, mock, mock2, mock3, mock4, mock5):
-			assert service_installer.install_zeppelin('../conf') == True
+			assert service_installer.install_zeppelin() == True
 
 
 	@mock.patch('scripts.service_installer.is_ambari_installed', return_value=True)
@@ -136,7 +136,7 @@ def test_zeppelin_check_is_good(self, mock, mock2, mock3, mock4, mock5):
 	@mock.patch('scripts.service_installer.check_ambari_service_installed', return_value=False)
 	@mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'y'])
 	def test_zeppelin_no_ambari_contact_continue(self, mock, mock2, mock3, mock4, mock5):
-			assert service_installer.install_zeppelin('../conf') == True
+			assert service_installer.install_zeppelin() == True
 
 
 	@mock.patch('scripts.service_installer.is_ambari_installed', return_value=True)
@@ -145,28 +145,28 @@ def test_zeppelin_no_ambari_contact_continue(self, mock, mock2, mock3, mock4, mo
 	@mock.patch('scripts.service_installer.check_ambari_service_installed', return_value=False)
 	@mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'n'])
 	def test_zeppelin_no_ambari_contact_no_continue(self, mock, mock2, mock3, mock4, mock5):
-			assert service_installer.install_zeppelin('../conf') == False
+			assert service_installer.install_zeppelin() == False
 
 class TestAmbariServiceCheck(unittest.TestCase):
 
 
 	@mock.patch('scripts.curl_client.CurlClient.make_request', side_effect=[['', ''], ['200 OK', '']])
 	@mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'n'])
 	def test_ambari_check_good(self, mock, mock2):
-		conf = scripts.config.read_config('../conf/global-config.conf')['AMBARI']
+		conf = scripts.config.read_config('global-config.conf')['AMBARI']
 		assert service_installer.check_ambari_service_installed('ZEPPELIN', conf) == True
 
 	@mock.patch('scripts.curl_client.CurlClient.make_request', side_effect=[['200 OK', ''], ['200 OK', '']])
 	@mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'n'])
 	def test_ambari_check_false(self, mock, mock2):
-		conf = scripts.config.read_config('../conf/global-config.conf')['AMBARI']
+		conf = scripts.config.read_config('global-config.conf')['AMBARI']
 		assert service_installer.check_ambari_service_installed('ZEPPELIN', conf) == True
 
 
 	@mock.patch('scripts.curl_client.CurlClient.make_request', side_effect=[['', ''],['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', '']])
 	@mock.patch('__builtin__.raw_input', side_effect=['', '', '', '', '', '', '', '', '', '', ''])
 	def test_ambari_check_many_attempts(self, mock, mock2):
-		conf = scripts.config.read_config('../conf/global-config.conf')['AMBARI']
+		conf = scripts.config.read_config('global-config.conf')['AMBARI']
 		assert service_installer.check_ambari_service_installed('ZEPPELIN', conf) == False
 
 
@@ -206,7 +206,6 @@ def test_nifi_check_is_good(self, mock, mock2, mock3, mock4, mock5):
 	@mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'y'])
 	def test_nifi_no_ambari_contact_continue(self, mock, mock2, mock3, mock4, mock5):
 			assert service_installer.install_nifi('../conf') == True
-
 
 	@mock.patch('scripts.service_installer.is_ambari_installed', return_value=True)
 	@mock.patch('scripts.service_installer.is_hdp_select_installed', return_value=True)
@@ -215,39 +214,45 @@ def test_nifi_no_ambari_contact_continue(self, mock, mock2, mock3, mock4, mock5)
 	@mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'n'])
 	def test_nifi_no_ambari_contact_no_continue(self, mock, mock2, mock3, mock4, mock5):
 			assert service_installer.install_nifi('../conf') == False
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+class TestZeppelinAddNotebook(unittest.TestCase):
+
+	@mock.patch('scripts.curl_client.CurlClient.make_request', return_value=['201 Created', ''])
+	@mock.patch('os.path.isfile', return_value=True)
+	@mock.patch('os.listdir', return_value=['note1.json', 'note2.json', 'note3.json'])
+	def test_good(self, mock1, mock2, mock3):
+		assert (service_installer.add_zeppelin_notebooks())
+
+	@mock.patch('scripts.curl_client.CurlClient.make_request', return_value=['500 err', ''])
+	@mock.patch('os.path.isfile', return_value=True)
+	@mock.patch('os.listdir', return_value=['note1.json', 'note2.json', 'note3.json'])
+	def test_bad(self, mock1, mock2, mock3):
+		assert (not service_installer.add_zeppelin_notebooks())
+
+	@mock.patch('scripts.curl_client.CurlClient.make_request', return_value=['500 err', ''])
+	@mock.patch('os.path.isfile', return_value=True)
+	@mock.patch('os.listdir', return_value=['note1.json', 'note2.json', 'note3.xml'])
+	def test_mixed(self, mock1, mock2, mock3):
+		assert (not service_installer.add_zeppelin_notebooks())
+
+	@mock.patch('scripts.curl_client.CurlClient.make_request', side_effect=[['500 err', ''], ['201 Created', ''], ['201 Created', '']])
+	@mock.patch('os.path.isfile', return_value=True)
+	@mock.patch('os.listdir', return_value=['note1.json', 'note2.json', 'note3.xml'])
+	def test_mixed_response(self, mock1, mock2, mock3):
+		assert (not service_installer.add_zeppelin_notebooks())
+
+	@mock.patch('scripts.curl_client.CurlClient.make_request', side_effect=[['500 err', ''], ['201 Created', ''], ['201 Created', '']])
+	@mock.patch('os.path.isfile', return_value=True)
+	def test_post_notebook(self, mock1, mock2):
+
+		paths = ['note1.json', 'note2.json', 'note3.xml']
+		assert not service_installer.post_notebook(paths[0])
+
+		assert service_installer.post_notebook(paths[1])
+
+		assert service_installer.post_notebook(paths[2])
+
+
+