diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..669a2aa --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ambari-services/ambari-zeppelin-service"] + path = ambari-services/ambari-zeppelin-service + url = https://github.com/hortonworks-gallery/ambari-zeppelin-service diff --git a/ambari-services/ambari-zeppelin-service b/ambari-services/ambari-zeppelin-service new file mode 160000 index 0000000..8d5ce60 --- /dev/null +++ b/ambari-services/ambari-zeppelin-service @@ -0,0 +1 @@ +Subproject commit 8d5ce600bf8f22c71a041b3cf80732734affb57e diff --git a/conf/global-config.conf b/conf/global-config.conf new file mode 100644 index 0000000..db73524 --- /dev/null +++ b/conf/global-config.conf @@ -0,0 +1,16 @@ +[DEMO] +name="Solutions Demo" + +[AMBARI] +server=localhost +port=8080 +username=admin +password=admin +cluster_name=Sandbox +proto=http + +[ZEPPELIN] +notebooks_directory="/var/zeppelin/notebooks" + +[NIFI] +INSTALL_DIR="/opt" \ No newline at end of file diff --git a/conf/service-installer.conf b/conf/service-installer.conf new file mode 100644 index 0000000..9461aec --- /dev/null +++ b/conf/service-installer.conf @@ -0,0 +1,15 @@ +[SERVICES] +service-names=["ZEPPELIN"] + + +[ZEPPELIN] +install-commands=[ "hdp-select status hadoop-client | sed 's/hadoop-client - \\([0-9]\\.[0-9]\\).*/\\1/'", "cp -r ../ambari-services/ambari-zeppelin-service /var/lib/ambari-server/resources/stacks/HDP/$VERSION/services/ZEPPELIN", "ambari-server restart"] +server=localhost +port=9995 + +# Repo Links for HDP-SELECT +[HDP-SELECT] +ubuntu12="http://public-repo-1.hortonworks.com/HDP/ubuntu12/2.x/updates/2.4.2.0/pool/main/h/hdp-select/hdp-select_2.4.2.0-258_all.deb" +ubuntu14="http://public-repo-1.hortonworks.com/HDP/ubuntu14/2.x/updates/2.4.2.0/pool/main/h/hdp-select/hdp-select_2.4.2.0-258_all.deb" +centos6="http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.4.0.0/hdp-select/hdp-select-2.4.0.0-169.el6.noarch.rpm" +centos7="http://public-repo-1.hortonworks.com/HDP/centos7/2.x/updates/2.4.0.0/hdp-select/hdp-select-2.4.0.0-169.el6.noarch.rpm" diff --git a/scripts/config.properties b/scripts/config.properties deleted file mode 100644 index 07b263f..0000000 --- a/scripts/config.properties +++ /dev/null @@ -1,21 +0,0 @@ -[INSTALLER] -dir=/tmp/demo-installer - -[DEMO] -name="Solutions Demo" - - -[AMBARI] -host=sandbox.hortonworks.com -port=8080 -username=admin -password=admin -cluster_name=Sandbox - -# set hdp_version=`hdp-select status hadoop-client | sed 's/hadoop-client - \([0-9]\.[0-9]\).*/\1/'` - -[ZEPPELIN] -notebooks_directory="/var/zeppelin/notebooks" - -[NIFI] -INSTALL_DIR="/opt" \ No newline at end of file diff --git a/scripts/curl_client.py b/scripts/curl_client.py index 98628b0..30c4eef 100644 --- a/scripts/curl_client.py +++ b/scripts/curl_client.py @@ -28,10 +28,14 @@ def set_server(self, server): # A number between 0 and 65535 def set_port(self, port): - if not type(port) is int: + int_port = -1 + try: + int_port = int(port) + except ValueError as e: raise ValueError('Server port was not of type: int') - if port > 0 and port <= 65535: - self.port = port + + if int_port > 0 and int_port <= 65535: + self.port = int_port else: raise ValueError('Server port must be between 0 and 65535. Value was ' + str(port)) @@ -42,7 +46,7 @@ def set_port(self, port): # A list of query parameters # ['param1=value1', 'param2=value2'] - def make_request(self, verb, request, query=''): + def make_request(self, verb, request, options='', query=''): if not (verb == 'GET' or verb == 'POST' or verb == 'PUT' or verb == 'DELETE'): raise ValueError('HTTP Verb must be one of GET|PUT|POST|DELETE') @@ -58,7 +62,7 @@ def make_request(self, verb, request, query=''): method = '-X ' + verb - call = ' '.join(['curl -sS', credentials, method, url]) + call = ' '.join(['curl -sS', credentials, method, options, url]) output = self.cmd.run(call) return output diff --git a/scripts/service_installer.py b/scripts/service_installer.py new file mode 100644 index 0000000..a016d0f --- /dev/null +++ b/scripts/service_installer.py @@ -0,0 +1,160 @@ +# Script which installs Zeppelin as an Ambari Service +import config, sys, platform, json, time +from shell import Shell +from curl_client import CurlClient + +def install_hdp_select(): + dist_info = platform.linux_distribution() + if(len(dist_info[0]) == 0): + raise EnvironmentError('You must be running a linux distribution to install hdp-select') + +# Only want to get distro name +# Determine first distro name +# Then determine the version (first char char for centos ) + distro = dist_info[0].lower() + fullname = distro + ver = dist_info[1] + if 'centos' in distro: # Get First 1/2 nums in version string + fullname = fullname + dist_info[1][0] + elif 'ubuntu' in distro: + if (len(dist_info[1]) < 2): + fullname = fullname + dist_info[1][0] + else: + fullname = fullname + dist_info[1][0] + dist_info[1][1] + + conf = config.read_config('../conf/service-installer.conf') + urls = conf['HDP-SELECT'] + url = '' + if fullname == 'centos6': + url = urls['centos6'] + elif fullname == 'centos7': + url = urls['centos7'] + elif fullname == 'ubuntu12': + url = urls['ubuntu12'] + elif fullname == 'ubuntu14': + url = urls['ubuntu14'] + + res = '' + + if len(url) == 0: + raise EnvironmentError('Must be using one of: CentOS 6.x, CentOS 7.x, Ubuntu 12.x, Ubuntu 14.x') + elif 'centos' in fullname: + cmd = Shell() + output = cmd.run('yum install -y ' + url) + res = cmd.run('which hdp-select') + elif 'ubuntu' in fullname: + cmd = Shell() + output = cmd.run('wget ' + url + ' -O ./hdp-select.deb') + output = cmd.run('dpkg -i ' + './hdp-select.deb') + res = cmd.run('which hdp-select') + + if len(res[0]) == 0: + return False + else: + return True + +def is_hdp_select_installed(): + sh = Shell() + output = sh.run('which hdp-select') + + if len(output[0]) == 0: + return False + else: + return True + +def is_ambari_installed(): + sh = Shell() + output = sh.run('which ambari-server') + if len(output[0]) == 0: + return False + else: + return True + + + +def install_zeppelin(conf_dir): + + if not conf_dir.endswith('/'): + conf_dir += '/' + + if not is_ambari_installed(): + raise EnvironmentError('You must install the demo on the same node as the Ambari server. Install Ambari here or move to another node with Ambari installed before continuing') + + + if not is_hdp_select_installed(): + installed = install_hdp_select() + if not installed: + raise EnvironmentError('hdp-select could not be installed. Please install it manually and then re-run the setup.') + + conf = config.read_config(conf_dir + 'service-installer.conf') + cmds = conf['ZEPPELIN']['install-commands'] + cmds = json.loads(conf['ZEPPELIN']['install-commands']) + + sh = Shell() +# print(sh.run('pwd')[0]) + version = sh.run(cmds[0]) +# print("HDP-VERSION: " + str(version[0])) + fixed_cmd = cmds[1].replace('$VERSION', str(version[0])).replace('\n', '') +# print('FIXED COPY COMMAND: ' + fixed_cmd) + copy = sh.run(fixed_cmd) +# print("COPY OUTPUT: " + copy[0]) + restart = sh.run(cmds[2]) +# print("Restart output: " + restart[0]) + + + print("Please open the Ambari Interface and manually deploy the Zeppelin Service.") + raw_input("Press enter twice to continue...") + raw_input("Press enter once to continue...") + +# We've copied the necessary files. Once that completes we need to add it to Ambari + + print('Checking to make sure service is installed') + ambari = config.read_config(conf_dir + 'global-config.conf')['AMBARI'] + installed = check_ambari_service_installed('ZEPPELIN', ambari) + cont = '' + if not installed: + print('Unable to contact Ambari Server. Unsure whether or not Zeppelin was installed') + while not (cont == 'y' or cont == 'n'): + cont = raw_input('Continue attempt to set up Zeppelin for demo?(y/n)') + if not (cont == 'y' or cont == 'n'): + print('Please enter "y" or "n"') + else: + cont = 'y' + + if cont == 'n': + return False + elif cont == 'y': + return True + +def check_ambari_service_installed(service_name, ambari_config): + + curl = CurlClient(username=ambari_config['username'], password=ambari_config['password'], port=ambari_config['port'], server=ambari_config['server'], proto=ambari_config['proto']) + + cluster_name = ambari_config['cluster_name'] + request = '/api/v1/clusters/' + cluster_name + '/services/' + service_name + attempts = 0 + while attempts < 10: + output = curl.make_request('GET', request, '-i') + if '200 OK' in output[0]: + print('Service Installed Sucessfully') + return True + else: + attempts += 1 + raw_input('Could not connect.' + str(10-attempts) + ' remaining. Press any key to continue') + + return False + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/test_curlclient.py b/tests/test_curlclient.py index 310202f..b466dcd 100644 --- a/tests/test_curlclient.py +++ b/tests/test_curlclient.py @@ -10,9 +10,9 @@ resErr = ['', '\'msg\':\'err\''] def mocked_request(*args, **kwargs): - if args[0] == 'curl -sS -u admin:admin -X GET http://demo-server:9090/api/v1/test?': + if '/api/v1/test' in args[0]: return res1 - elif args[0] == 'curl -sS -u admin:admin -X GET http://demo-server:9090/api/v1/bad?': + elif '/api/v1/bad' in args[0]: return res2 else: return res3 diff --git a/tests/test_service_installer.py b/tests/test_service_installer.py new file mode 100644 index 0000000..f99455d --- /dev/null +++ b/tests/test_service_installer.py @@ -0,0 +1,207 @@ +import unittest, json, mock, scripts +from env import scripts +from mock import Mock +from scripts import service_installer + +non_linux_distro = ['', '', ''] +other_linux_distro = ['mint', '', ''] +centos_6_distro = ['CentOS', '6.8', 'Final'] +centos_7_distro = ['CentOS', '7.1', 'Final'] +ubuntu_12_distro = ['Ubuntu', '12.04', 'precise'] +ubuntu_14_distro = ['Ubuntu', '14.04', 'trusty'] +bad_ubuntu = ['Ubuntu', '1', 'NaN'] + +class TestHDPSelectInstall(unittest.TestCase): + + @mock.patch('platform.linux_distribution', return_value=centos_6_distro) + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) + def test_hdp_select_centos_6_pass(self, mock, mock2): + assert True == service_installer.install_hdp_select() + + + @mock.patch('platform.linux_distribution', return_value=centos_7_distro) + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) + def test_hdp_select_centos_7_pass(self, mock, mock2): + assert True == service_installer.install_hdp_select() + + @mock.patch('platform.linux_distribution', return_value=ubuntu_12_distro) + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) + def test_hdp_select_ubuntu_12_pass(self, mock, mock2): + assert True == service_installer.install_hdp_select() + + @mock.patch('platform.linux_distribution', return_value=ubuntu_14_distro) + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) + def test_hdp_select_ubuntu_14_pass(self, mock, mock2): + assert True == service_installer.install_hdp_select() + + + @mock.patch('platform.linux_distribution', return_value=centos_6_distro) + @mock.patch('scripts.shell.Shell.run', return_value=['', '']) + def test_hdp_select_centos_6_fail(self, mock, mock2): + assert False == service_installer.install_hdp_select() + + @mock.patch('platform.linux_distribution', return_value=centos_7_distro) + @mock.patch('scripts.shell.Shell.run', return_value=['', '']) + def test_hdp_select_centos_7_fail(self, mock, mock2): + assert False == service_installer.install_hdp_select() + + @mock.patch('platform.linux_distribution', return_value=ubuntu_12_distro) + @mock.patch('scripts.shell.Shell.run', return_value=['', '']) + def test_hdp_select_ubuntu_12_fail(self, mock, mock2): + assert False == service_installer.install_hdp_select() + + @mock.patch('platform.linux_distribution', return_value=ubuntu_14_distro) + @mock.patch('scripts.shell.Shell.run', return_value=['', '']) + def test_hdp_select_ubuntu_14_fail(self, mock, mock2): + assert False == service_installer.install_hdp_select() + + @mock.patch('platform.linux_distribution', return_value=bad_ubuntu) + @mock.patch('scripts.shell.Shell.run', return_value=['', '']) + def test_hdp_select_bad_ubuntu(self, mock, mock2): + try: + service_installer.install_hdp_select() + self.fail('Should fail with a non-linux operating system') + except EnvironmentError as e: + assert str(e.message) == 'Must be using one of: CentOS 6.x, CentOS 7.x, Ubuntu 12.x, Ubuntu 14.x' + + + @mock.patch('platform.linux_distribution', return_value=non_linux_distro) + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) + def test_hdp_select_non_linux(self, mock, mock2): + try: + service_installer.install_hdp_select() + self.fail('Should fail with a non-linux operating system') + except EnvironmentError as e: + assert str(e.message) == 'You must be running a linux distribution to install hdp-select' + + @mock.patch('platform.linux_distribution', return_value=other_linux_distro) + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) + def test_hdp_select_other_linux(self, mock, mock2): + try: + service_installer.install_hdp_select() + self.fail('Should fail with a non-linux operating system') + except EnvironmentError as e: + assert str(e.message) == 'Must be using one of: CentOS 6.x, CentOS 7.x, Ubuntu 12.x, Ubuntu 14.x' + +class TestComponentCheck(unittest.TestCase): + + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) + def test_hdp_select_good(self, mock): + assert service_installer.is_hdp_select_installed() == True + + @mock.patch('scripts.shell.Shell.run', return_value=['', '']) + def test_hdp_select_bad(self, mock): + assert service_installer.is_hdp_select_installed() == False + + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/ambari-server', '']) + def test_ambari_good(self, mock): + assert service_installer.is_ambari_installed() == True + + @mock.patch('scripts.shell.Shell.run', return_value=['', '']) + def test_ambari_bad(self, mock): + assert service_installer.is_ambari_installed() == False + +class TestZeppelinInstall(unittest.TestCase): + + @mock.patch('scripts.service_installer.is_ambari_installed', return_value=False) + def test_zeppelin_ambari_bad(self, mock): + try: + service_installer.install_zeppelin('../conf/service-installer.conf') + self.fail('Cannot continue installation without Ambari') + except EnvironmentError as e: + assert str(e.message) == 'You must install the demo on the same node as the Ambari server. Install Ambari here or move to another node with Ambari installed before continuing' + + @mock.patch('scripts.service_installer.is_ambari_installed', return_value=True) + @mock.patch('scripts.service_installer.is_hdp_select_installed', return_value=False) + @mock.patch('scripts.service_installer.install_hdp_select', return_value=False) + def test_zeppelin_ambari_good(self, mock, mock2, mock3): #Also HDP select bad + try: + service_installer.install_zeppelin('../conf/service-installer.conf') + self.fail('Cannot continue installation without hdp-select') + except EnvironmentError as e: + assert str(e.message) == 'hdp-select could not be installed. Please install it manually and then re-run the setup.' + + @mock.patch('scripts.service_installer.is_ambari_installed', return_value=True) + @mock.patch('scripts.service_installer.is_hdp_select_installed', return_value=True) + @mock.patch('scripts.service_installer.install_hdp_select', return_value=True) + @mock.patch('scripts.service_installer.check_ambari_service_installed', return_value=True) + @mock.patch('__builtin__.raw_input', return_value='y') + def test_zeppelin_check_is_good(self, mock, mock2, mock3, mock4, mock5): + assert service_installer.install_zeppelin('../conf') == True + + + @mock.patch('scripts.service_installer.is_ambari_installed', return_value=True) + @mock.patch('scripts.service_installer.is_hdp_select_installed', return_value=True) + @mock.patch('scripts.service_installer.install_hdp_select', return_value=True) + @mock.patch('scripts.service_installer.check_ambari_service_installed', return_value=False) + @mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'y']) + def test_zeppelin_no_ambari_contact_continue(self, mock, mock2, mock3, mock4, mock5): + assert service_installer.install_zeppelin('../conf') == True + + + @mock.patch('scripts.service_installer.is_ambari_installed', return_value=True) + @mock.patch('scripts.service_installer.is_hdp_select_installed', return_value=True) + @mock.patch('scripts.service_installer.install_hdp_select', return_value=True) + @mock.patch('scripts.service_installer.check_ambari_service_installed', return_value=False) + @mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'n']) + def test_zeppelin_no_ambari_contact_no_continue(self, mock, mock2, mock3, mock4, mock5): + assert service_installer.install_zeppelin('../conf') == False + +class TestAmbariServiceCheck(unittest.TestCase): + + + @mock.patch('scripts.curl_client.CurlClient.make_request', side_effect=[['', ''], ['200 OK', '']]) + @mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'n']) + def test_ambari_check_good(self, mock, mock2): + conf = scripts.config.read_config('../conf/global-config.conf')['AMBARI'] + assert service_installer.check_ambari_service_installed('ZEPPELIN', conf) == True + + @mock.patch('scripts.curl_client.CurlClient.make_request', side_effect=[['200 OK', ''], ['200 OK', '']]) + @mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'n']) + def test_ambari_check_false(self, mock, mock2): + conf = scripts.config.read_config('../conf/global-config.conf')['AMBARI'] + assert service_installer.check_ambari_service_installed('ZEPPELIN', conf) == True + + + @mock.patch('scripts.curl_client.CurlClient.make_request', side_effect=[['', ''],['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', '']]) + @mock.patch('__builtin__.raw_input', side_effect=['', '', '', '', '', '', '', '', '', '', '']) + def test_ambari_check_many_attempts(self, mock, mock2): + conf = scripts.config.read_config('../conf/global-config.conf')['AMBARI'] + assert service_installer.check_ambari_service_installed('ZEPPELIN', conf) == False + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file