From 045e559d5c930366a4dabdc4c98510192998c714 Mon Sep 17 00:00:00 2001 From: zblanco Date: Tue, 21 Jun 2016 13:01:24 -0700 Subject: [PATCH 1/7] [Zeppelin Ambari Service] - Adding submodule from hw gallery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I decided to use a git submodule to house the amber service so that it’s possible to update and I also don’t have to create the service from scratch --- .gitmodules | 3 +++ ambari-services/ambari-zeppelin-service | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 ambari-services/ambari-zeppelin-service diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..669a2aa --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ambari-services/ambari-zeppelin-service"] + path = ambari-services/ambari-zeppelin-service + url = https://github.com/hortonworks-gallery/ambari-zeppelin-service diff --git a/ambari-services/ambari-zeppelin-service b/ambari-services/ambari-zeppelin-service new file mode 160000 index 0000000..8d5ce60 --- /dev/null +++ b/ambari-services/ambari-zeppelin-service @@ -0,0 +1 @@ +Subproject commit 8d5ce600bf8f22c71a041b3cf80732734affb57e From 4926210f4b3fe12f8dda6b9ac1a474a309c9bb70 Mon Sep 17 00:00:00 2001 From: zblanco Date: Wed, 22 Jun 2016 10:56:48 -0700 Subject: [PATCH 2/7] adding hep-select installer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this is required by the ambari services submodules that we’ll be using so it’s necessary that it is installed --- .../global-config.conf | 0 conf/service-installer.conf | 9 ++ scripts/service_installer.py | 90 +++++++++++ tests/test_service_installer.py | 147 ++++++++++++++++++ 4 files changed, 246 insertions(+) rename scripts/config.properties => conf/global-config.conf (100%) create mode 100644 conf/service-installer.conf create mode 100644 scripts/service_installer.py create mode 100644 tests/test_service_installer.py diff --git a/scripts/config.properties b/conf/global-config.conf similarity index 100% rename from scripts/config.properties rename to conf/global-config.conf diff --git a/conf/service-installer.conf b/conf/service-installer.conf new file mode 100644 index 0000000..0cf19b3 --- /dev/null +++ b/conf/service-installer.conf @@ -0,0 +1,9 @@ +[ZEPPELIN] +install-commands=[ "export VERSION=`hdp-select status hadoop-client | sed 's/hadoop-client - \\([0-9]\\.[0-9]\\).*/\\1/'`", "cp -r ambari-services/ambari-zeppelin-service /var/lib/ambari-server/resources/stacks/HDP/$VERSION/services/ZEPPELIN", "ambari-server restart"] + +# Repo Links for HDP-SELECT +[HDP-SELECT] +ubuntu12="http://public-repo-1.hortonworks.com/HDP/ubuntu12/2.x/updates/2.4.2.0/pool/main/h/hdp-select/hdp-select_2.4.2.0-258_all.deb" +ubuntu14="http://public-repo-1.hortonworks.com/HDP/ubuntu14/2.x/updates/2.4.2.0/pool/main/h/hdp-select/hdp-select_2.4.2.0-258_all.deb" +centos6="http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.4.0.0/hdp-select/hdp-select-2.4.0.0-169.el6.noarch.rpm" +centos7="http://public-repo-1.hortonworks.com/HDP/centos7/2.x/updates/2.4.0.0/hdp-select/hdp-select-2.4.0.0-169.el6.noarch.rpm" diff --git a/scripts/service_installer.py b/scripts/service_installer.py new file mode 100644 index 0000000..1b8af93 --- /dev/null +++ b/scripts/service_installer.py @@ -0,0 +1,90 @@ +# Script which installs Zeppelin as an Ambari Service +import config, sys, platform, json +from shell import Shell + +def install_hdp_select(): + dist_info = platform.linux_distribution() + if(len(dist_info[0]) == 0): + raise EnvironmentError('You must be running a linux distribution to install hdp-select') + +# Only want to get distro name +# Determine first distro name +# Then determine the version (first char char for centos ) + distro = dist_info[0].lower() + fullname = distro + ver = dist_info[1] + if 'centos' in distro: # Get First 1/2 nums in version string + fullname = fullname + dist_info[1][0] + elif 'ubuntu' in distro: + if len(dist_info[1]) < 2: + fullname = fullname + dist_info[1][0] + else: + fullname = fullname + dist_info[1][0] + dist_info[1][1] + + conf = config.read_config('../conf/service-installer.conf') + urls = conf['HDP-SELECT'] + url = '' + if fullname == 'centos6': + url = urls['centos6'] + elif fullname == 'centos7': + url = urls['centos7'] + elif fullname == 'ubuntu12': + url = urls['ubuntu12'] + elif fullname == 'ubuntu14': + url = urls['ubuntu14'] + + res = '' + + if len(url) == 0: + raise EnvironmentError('Must be using one of: CentOS 6.x, CentOS 7.x, Ubuntu 12.x, Ubuntu 14.x') + elif 'centos' in fullname: + cmd = Shell() + output = cmd.run('yum install -y ' + url) + res = cmd.run('which hdp-select') + elif 'ubuntu' in fullname: + cmd = Shell() + output = cmd.run('wget ' + url + ' -O ./hdp-select.deb') + output = cmd.run('dpkg -i ' + './hdp-select.deb') + res = cmd.run('which hdp-select') + + if len(res) == 0: + return False + else: + return True + +def is_hdp_select_installed(): + sh = Shell() + output = sh.run('which hdp-select') + if len(output) == 0: + return False + else: + return True + +#def install_zeppelin(conf_file): +# install +# conf = config.read_config(conf_file) +# cmds = conf['ZEPPELIN']['install-commands'] +# cmds = json.loads(conf['ZEPPELIN']['install-commands']) +# sh = Shell() +# for cmd in cmds: +# print(cmd) +# print('') +# print(sh.run(cmd)) + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/test_service_installer.py b/tests/test_service_installer.py new file mode 100644 index 0000000..90f5db3 --- /dev/null +++ b/tests/test_service_installer.py @@ -0,0 +1,147 @@ +import unittest, json, mock +from env import scripts +from mock import Mock +from scripts import service_installer + +non_linux_distro = ['', '', ''] +other_linux_distro = ['mint', '', ''] +centos_6_distro = ['CentOS', '6.8', 'Final'] +centos_7_distro = ['CentOS', '6.8', 'Final'] +ubuntu_12_distro = ['Ubuntu', '12.04', 'precise'] +ubuntu_14_distro = ['Ubuntu', '14.04', 'trusty'] +bad_ubuntu = ['Ubuntu', '1', 'NaN'] + +#def mocked_cmd(*args, **kwargs): +# print("ARGS: " + ', '.join(args)) +# +# if 'centos6' in args[0]: +# return centos_6_distro +# elif 'centos7' in args[0]: +# return centos_7_distro +# elif 'ubuntu12' in args[0]: +# return ubuntu_12_distro +# elif 'ubuntu14' in args[0]: +# return ubuntu_14_distro +# elif 'nonlinux' in args[0]: +# return non_linux_distro +# + +class TestHDPSelectInstall(unittest.TestCase): + + @mock.patch('platform.linux_distribution', return_value=centos_6_distro) + @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') + def test_hdp_select_centos_6(self, mock, mock2): + assert True == service_installer.install_hdp_select() + + + @mock.patch('platform.linux_distribution', return_value=centos_7_distro) + @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') + def test_hdp_select_centos_7(self, mock, mock2): + assert True == service_installer.install_hdp_select() + + @mock.patch('platform.linux_distribution', return_value=ubuntu_12_distro) + @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') + def test_hdp_select_ubuntu_12(self, mock, mock2): + assert True == service_installer.install_hdp_select() + + @mock.patch('platform.linux_distribution', return_value=ubuntu_14_distro) + @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') + def test_hdp_select_ubuntu_14(self, mock, mock2): + assert True == service_installer.install_hdp_select() + + + @mock.patch('platform.linux_distribution', return_value=centos_6_distro) + @mock.patch('scripts.shell.Shell.run', return_value='') + def test_hdp_select_centos_6(self, mock, mock2): + assert False == service_installer.install_hdp_select() + + @mock.patch('platform.linux_distribution', return_value=centos_7_distro) + @mock.patch('scripts.shell.Shell.run', return_value='') + def test_hdp_select_centos_7(self, mock, mock2): + assert False == service_installer.install_hdp_select() + + @mock.patch('platform.linux_distribution', return_value=ubuntu_12_distro) + @mock.patch('scripts.shell.Shell.run', return_value='') + def test_hdp_select_ubuntu_12(self, mock, mock2): + assert False == service_installer.install_hdp_select() + + @mock.patch('platform.linux_distribution', return_value=ubuntu_14_distro) + @mock.patch('scripts.shell.Shell.run', return_value='') + def test_hdp_select_ubuntu_14(self, mock, mock2): + assert False == service_installer.install_hdp_select() + + @mock.patch('platform.linux_distribution', return_value=bad_ubuntu) + @mock.patch('scripts.shell.Shell.run', return_value='') + def test_hdp_select_ubuntu_14(self, mock, mock2): + try: + service_installer.install_hdp_select() + self.fail('Should fail with a non-linux operating system') + except EnvironmentError as e: + assert str(e.message) == 'Must be using one of: CentOS 6.x, CentOS 7.x, Ubuntu 12.x, Ubuntu 14.x' + + + @mock.patch('platform.linux_distribution', return_value=non_linux_distro) + @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') + def test_hdp_select_non_linux(self, mock, mock2): + try: + service_installer.install_hdp_select() + self.fail('Should fail with a non-linux operating system') + except EnvironmentError as e: + assert str(e.message) == 'You must be running a linux distribution to install hdp-select' + + @mock.patch('platform.linux_distribution', return_value=other_linux_distro) + @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') + def test_hdp_select_other_linux(self, mock, mock2): + try: + service_installer.install_hdp_select() + self.fail('Should fail with a non-linux operating system') + except EnvironmentError as e: + assert str(e.message) == 'Must be using one of: CentOS 6.x, CentOS 7.x, Ubuntu 12.x, Ubuntu 14.x' + +class TestHDPSelectCheck(unittest.TestCase): + + @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') + def test_hdp_select_good(self, mock): + assert service_installer.is_hdp_select_installed() == True + + @mock.patch('scripts.shell.Shell.run', return_value='') + def test_hdp_select_bad(self, mock): + assert service_installer.is_hdp_select_installed() == False + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 643637f9eade5173f8b2d28013cf4b010b8f00aa Mon Sep 17 00:00:00 2001 From: zblanco Date: Wed, 22 Jun 2016 11:43:05 -0700 Subject: [PATCH 3/7] better code coverage for hdp-select installer --- conf/global-config.conf | 3 --- conf/service-installer.conf | 2 +- scripts/service_installer.py | 16 +++------------ tests/test_service_installer.py | 35 ++++++++++----------------------- 4 files changed, 14 insertions(+), 42 deletions(-) diff --git a/conf/global-config.conf b/conf/global-config.conf index 07b263f..b6ca757 100644 --- a/conf/global-config.conf +++ b/conf/global-config.conf @@ -1,6 +1,3 @@ -[INSTALLER] -dir=/tmp/demo-installer - [DEMO] name="Solutions Demo" diff --git a/conf/service-installer.conf b/conf/service-installer.conf index 0cf19b3..d6b2e93 100644 --- a/conf/service-installer.conf +++ b/conf/service-installer.conf @@ -1,5 +1,5 @@ [ZEPPELIN] -install-commands=[ "export VERSION=`hdp-select status hadoop-client | sed 's/hadoop-client - \\([0-9]\\.[0-9]\\).*/\\1/'`", "cp -r ambari-services/ambari-zeppelin-service /var/lib/ambari-server/resources/stacks/HDP/$VERSION/services/ZEPPELIN", "ambari-server restart"] +install-commands=[ "VERSION=`hdp-select status hadoop-client | sed 's/hadoop-client - \\([0-9]\\.[0-9]\\).*/\\1/'`", "cp -r ambari-services/ambari-zeppelin-service /var/lib/ambari-server/resources/stacks/HDP/$VERSION/services/ZEPPELIN", "ambari-server restart"] # Repo Links for HDP-SELECT [HDP-SELECT] diff --git a/scripts/service_installer.py b/scripts/service_installer.py index 1b8af93..eeb8a89 100644 --- a/scripts/service_installer.py +++ b/scripts/service_installer.py @@ -16,11 +16,11 @@ def install_hdp_select(): if 'centos' in distro: # Get First 1/2 nums in version string fullname = fullname + dist_info[1][0] elif 'ubuntu' in distro: - if len(dist_info[1]) < 2: + if (len(dist_info[1]) < 2): fullname = fullname + dist_info[1][0] else: - fullname = fullname + dist_info[1][0] + dist_info[1][1] - + fullname = fullname + dist_info[1][0] + dist_info[1][1] + conf = config.read_config('../conf/service-installer.conf') urls = conf['HDP-SELECT'] url = '' @@ -60,16 +60,6 @@ def is_hdp_select_installed(): else: return True -#def install_zeppelin(conf_file): -# install -# conf = config.read_config(conf_file) -# cmds = conf['ZEPPELIN']['install-commands'] -# cmds = json.loads(conf['ZEPPELIN']['install-commands']) -# sh = Shell() -# for cmd in cmds: -# print(cmd) -# print('') -# print(sh.run(cmd)) diff --git a/tests/test_service_installer.py b/tests/test_service_installer.py index 90f5db3..b69e3af 100644 --- a/tests/test_service_installer.py +++ b/tests/test_service_installer.py @@ -6,73 +6,58 @@ non_linux_distro = ['', '', ''] other_linux_distro = ['mint', '', ''] centos_6_distro = ['CentOS', '6.8', 'Final'] -centos_7_distro = ['CentOS', '6.8', 'Final'] +centos_7_distro = ['CentOS', '7.1', 'Final'] ubuntu_12_distro = ['Ubuntu', '12.04', 'precise'] ubuntu_14_distro = ['Ubuntu', '14.04', 'trusty'] bad_ubuntu = ['Ubuntu', '1', 'NaN'] -#def mocked_cmd(*args, **kwargs): -# print("ARGS: " + ', '.join(args)) -# -# if 'centos6' in args[0]: -# return centos_6_distro -# elif 'centos7' in args[0]: -# return centos_7_distro -# elif 'ubuntu12' in args[0]: -# return ubuntu_12_distro -# elif 'ubuntu14' in args[0]: -# return ubuntu_14_distro -# elif 'nonlinux' in args[0]: -# return non_linux_distro -# - class TestHDPSelectInstall(unittest.TestCase): @mock.patch('platform.linux_distribution', return_value=centos_6_distro) @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') - def test_hdp_select_centos_6(self, mock, mock2): + def test_hdp_select_centos_6_good_pass(self, mock, mock2): assert True == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=centos_7_distro) @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') - def test_hdp_select_centos_7(self, mock, mock2): + def test_hdp_select_centos_7_pass(self, mock, mock2): assert True == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=ubuntu_12_distro) @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') - def test_hdp_select_ubuntu_12(self, mock, mock2): + def test_hdp_select_ubuntu_12_pass(self, mock, mock2): assert True == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=ubuntu_14_distro) @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') - def test_hdp_select_ubuntu_14(self, mock, mock2): + def test_hdp_select_ubuntu_14_pass(self, mock, mock2): assert True == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=centos_6_distro) @mock.patch('scripts.shell.Shell.run', return_value='') - def test_hdp_select_centos_6(self, mock, mock2): + def test_hdp_select_centos_6_fail(self, mock, mock2): assert False == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=centos_7_distro) @mock.patch('scripts.shell.Shell.run', return_value='') - def test_hdp_select_centos_7(self, mock, mock2): + def test_hdp_select_centos_7_fail(self, mock, mock2): assert False == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=ubuntu_12_distro) @mock.patch('scripts.shell.Shell.run', return_value='') - def test_hdp_select_ubuntu_12(self, mock, mock2): + def test_hdp_select_ubuntu_12_fail(self, mock, mock2): assert False == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=ubuntu_14_distro) @mock.patch('scripts.shell.Shell.run', return_value='') - def test_hdp_select_ubuntu_14(self, mock, mock2): + def test_hdp_select_ubuntu_14_fail(self, mock, mock2): assert False == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=bad_ubuntu) @mock.patch('scripts.shell.Shell.run', return_value='') - def test_hdp_select_ubuntu_14(self, mock, mock2): + def test_hdp_select_bad_ubuntu(self, mock, mock2): try: service_installer.install_hdp_select() self.fail('Should fail with a non-linux operating system') From 511c74a3d7a049604906d2a7a08c1a6d99faf47d Mon Sep 17 00:00:00 2001 From: zblanco Date: Wed, 22 Jun 2016 11:56:59 -0700 Subject: [PATCH 4/7] fixing output collection for real MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Didn’t use the array out output for stderr and stdout before --- scripts/service_installer.py | 35 +++++++++++++++++++++++++++++++-- tests/test_service_installer.py | 29 ++++++++++++++------------- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/scripts/service_installer.py b/scripts/service_installer.py index eeb8a89..17fe1d3 100644 --- a/scripts/service_installer.py +++ b/scripts/service_installer.py @@ -47,7 +47,7 @@ def install_hdp_select(): output = cmd.run('dpkg -i ' + './hdp-select.deb') res = cmd.run('which hdp-select') - if len(res) == 0: + if len(res[0]) == 0: return False else: return True @@ -55,11 +55,42 @@ def install_hdp_select(): def is_hdp_select_installed(): sh = Shell() output = sh.run('which hdp-select') - if len(output) == 0: + if len(output[0]) == 0: return False else: return True +def is_ambari_installed(): + sh = Shell() + output = sh.run('which ambari-server') + if len(output[0]) == 0: + return False + else: + return True + + +def install_zeppelin(conf_file): + + if not is_ambari_installed(): + raise EnvironmentError('You must install the demo on the same node as the Ambari server. Install Ambari here or move to another node with Ambari installed before continuing') + + + if not is_hdp_select_installed(): + installed = install_hdp_select() + if not installed: + raise EnvironmentError('hdp-select could not be installed. Please install it manually and then re-run the setup.') + + conf = config.read_config(conf_file) + cmds = conf['ZEPPELIN']['install-commands'] + cmds = json.loads(conf['ZEPPELIN']['install-commands']) + + sh = Shell() + version = sh.run(cmds[0]) + + fixed_cmd = cmds[1].replace('$VERSION', str(version)) + copy = sh.run(fixed_ver) + + restart = sh.run(cmds[2]) diff --git a/tests/test_service_installer.py b/tests/test_service_installer.py index b69e3af..5e16eec 100644 --- a/tests/test_service_installer.py +++ b/tests/test_service_installer.py @@ -14,49 +14,49 @@ class TestHDPSelectInstall(unittest.TestCase): @mock.patch('platform.linux_distribution', return_value=centos_6_distro) - @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') - def test_hdp_select_centos_6_good_pass(self, mock, mock2): + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) + def test_hdp_select_centos_6_pass(self, mock, mock2): assert True == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=centos_7_distro) - @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) def test_hdp_select_centos_7_pass(self, mock, mock2): assert True == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=ubuntu_12_distro) - @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) def test_hdp_select_ubuntu_12_pass(self, mock, mock2): assert True == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=ubuntu_14_distro) - @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) def test_hdp_select_ubuntu_14_pass(self, mock, mock2): assert True == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=centos_6_distro) - @mock.patch('scripts.shell.Shell.run', return_value='') + @mock.patch('scripts.shell.Shell.run', return_value=['', '']) def test_hdp_select_centos_6_fail(self, mock, mock2): assert False == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=centos_7_distro) - @mock.patch('scripts.shell.Shell.run', return_value='') + @mock.patch('scripts.shell.Shell.run', return_value=['', '']) def test_hdp_select_centos_7_fail(self, mock, mock2): assert False == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=ubuntu_12_distro) - @mock.patch('scripts.shell.Shell.run', return_value='') + @mock.patch('scripts.shell.Shell.run', return_value=['', '']) def test_hdp_select_ubuntu_12_fail(self, mock, mock2): assert False == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=ubuntu_14_distro) - @mock.patch('scripts.shell.Shell.run', return_value='') + @mock.patch('scripts.shell.Shell.run', return_value=['', '']) def test_hdp_select_ubuntu_14_fail(self, mock, mock2): assert False == service_installer.install_hdp_select() @mock.patch('platform.linux_distribution', return_value=bad_ubuntu) - @mock.patch('scripts.shell.Shell.run', return_value='') + @mock.patch('scripts.shell.Shell.run', return_value=['', '']) def test_hdp_select_bad_ubuntu(self, mock, mock2): try: service_installer.install_hdp_select() @@ -66,7 +66,7 @@ def test_hdp_select_bad_ubuntu(self, mock, mock2): @mock.patch('platform.linux_distribution', return_value=non_linux_distro) - @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) def test_hdp_select_non_linux(self, mock, mock2): try: service_installer.install_hdp_select() @@ -75,7 +75,7 @@ def test_hdp_select_non_linux(self, mock, mock2): assert str(e.message) == 'You must be running a linux distribution to install hdp-select' @mock.patch('platform.linux_distribution', return_value=other_linux_distro) - @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) def test_hdp_select_other_linux(self, mock, mock2): try: service_installer.install_hdp_select() @@ -85,13 +85,14 @@ def test_hdp_select_other_linux(self, mock, mock2): class TestHDPSelectCheck(unittest.TestCase): - @mock.patch('scripts.shell.Shell.run', return_value='/usr/bin/hdp-select') + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) def test_hdp_select_good(self, mock): assert service_installer.is_hdp_select_installed() == True - @mock.patch('scripts.shell.Shell.run', return_value='') + @mock.patch('scripts.shell.Shell.run', return_value=['', '']) def test_hdp_select_bad(self, mock): assert service_installer.is_hdp_select_installed() == False + From ea7433850ca62fae9d4d507c242cffbe6ed5212f Mon Sep 17 00:00:00 2001 From: zblanco Date: Thu, 23 Jun 2016 08:57:17 -0700 Subject: [PATCH 5/7] Updates to curl_client for service installer Needed to be able to add more arguments in --- conf/global-config.conf | 6 ++---- scripts/curl_client.py | 14 +++++++++----- tests/test_curlclient.py | 4 ++-- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/conf/global-config.conf b/conf/global-config.conf index b6ca757..db73524 100644 --- a/conf/global-config.conf +++ b/conf/global-config.conf @@ -1,15 +1,13 @@ [DEMO] name="Solutions Demo" - [AMBARI] -host=sandbox.hortonworks.com +server=localhost port=8080 username=admin password=admin cluster_name=Sandbox - -# set hdp_version=`hdp-select status hadoop-client | sed 's/hadoop-client - \([0-9]\.[0-9]\).*/\1/'` +proto=http [ZEPPELIN] notebooks_directory="/var/zeppelin/notebooks" diff --git a/scripts/curl_client.py b/scripts/curl_client.py index 98628b0..30c4eef 100644 --- a/scripts/curl_client.py +++ b/scripts/curl_client.py @@ -28,10 +28,14 @@ def set_server(self, server): # A number between 0 and 65535 def set_port(self, port): - if not type(port) is int: + int_port = -1 + try: + int_port = int(port) + except ValueError as e: raise ValueError('Server port was not of type: int') - if port > 0 and port <= 65535: - self.port = port + + if int_port > 0 and int_port <= 65535: + self.port = int_port else: raise ValueError('Server port must be between 0 and 65535. Value was ' + str(port)) @@ -42,7 +46,7 @@ def set_port(self, port): # A list of query parameters # ['param1=value1', 'param2=value2'] - def make_request(self, verb, request, query=''): + def make_request(self, verb, request, options='', query=''): if not (verb == 'GET' or verb == 'POST' or verb == 'PUT' or verb == 'DELETE'): raise ValueError('HTTP Verb must be one of GET|PUT|POST|DELETE') @@ -58,7 +62,7 @@ def make_request(self, verb, request, query=''): method = '-X ' + verb - call = ' '.join(['curl -sS', credentials, method, url]) + call = ' '.join(['curl -sS', credentials, method, options, url]) output = self.cmd.run(call) return output diff --git a/tests/test_curlclient.py b/tests/test_curlclient.py index 310202f..b466dcd 100644 --- a/tests/test_curlclient.py +++ b/tests/test_curlclient.py @@ -10,9 +10,9 @@ resErr = ['', '\'msg\':\'err\''] def mocked_request(*args, **kwargs): - if args[0] == 'curl -sS -u admin:admin -X GET http://demo-server:9090/api/v1/test?': + if '/api/v1/test' in args[0]: return res1 - elif args[0] == 'curl -sS -u admin:admin -X GET http://demo-server:9090/api/v1/bad?': + elif '/api/v1/bad' in args[0]: return res2 else: return res3 From 0eea83513c42798369155bf378a6ee8844833b02 Mon Sep 17 00:00:00 2001 From: zblanco Date: Thu, 23 Jun 2016 09:00:20 -0700 Subject: [PATCH 6/7] Adding Zeppelin installer function --- conf/service-installer.conf | 8 +++- scripts/service_installer.py | 63 ++++++++++++++++++++++++---- tests/test_service_installer.py | 73 +++++++++++++++++++++++++++++++-- 3 files changed, 133 insertions(+), 11 deletions(-) diff --git a/conf/service-installer.conf b/conf/service-installer.conf index d6b2e93..9461aec 100644 --- a/conf/service-installer.conf +++ b/conf/service-installer.conf @@ -1,5 +1,11 @@ +[SERVICES] +service-names=["ZEPPELIN"] + + [ZEPPELIN] -install-commands=[ "VERSION=`hdp-select status hadoop-client | sed 's/hadoop-client - \\([0-9]\\.[0-9]\\).*/\\1/'`", "cp -r ambari-services/ambari-zeppelin-service /var/lib/ambari-server/resources/stacks/HDP/$VERSION/services/ZEPPELIN", "ambari-server restart"] +install-commands=[ "hdp-select status hadoop-client | sed 's/hadoop-client - \\([0-9]\\.[0-9]\\).*/\\1/'", "cp -r ../ambari-services/ambari-zeppelin-service /var/lib/ambari-server/resources/stacks/HDP/$VERSION/services/ZEPPELIN", "ambari-server restart"] +server=localhost +port=9995 # Repo Links for HDP-SELECT [HDP-SELECT] diff --git a/scripts/service_installer.py b/scripts/service_installer.py index 17fe1d3..a016d0f 100644 --- a/scripts/service_installer.py +++ b/scripts/service_installer.py @@ -1,6 +1,7 @@ # Script which installs Zeppelin as an Ambari Service -import config, sys, platform, json +import config, sys, platform, json, time from shell import Shell +from curl_client import CurlClient def install_hdp_select(): dist_info = platform.linux_distribution() @@ -55,6 +56,7 @@ def install_hdp_select(): def is_hdp_select_installed(): sh = Shell() output = sh.run('which hdp-select') + if len(output[0]) == 0: return False else: @@ -69,7 +71,11 @@ def is_ambari_installed(): return True -def install_zeppelin(conf_file): + +def install_zeppelin(conf_dir): + + if not conf_dir.endswith('/'): + conf_dir += '/' if not is_ambari_installed(): raise EnvironmentError('You must install the demo on the same node as the Ambari server. Install Ambari here or move to another node with Ambari installed before continuing') @@ -80,20 +86,63 @@ def install_zeppelin(conf_file): if not installed: raise EnvironmentError('hdp-select could not be installed. Please install it manually and then re-run the setup.') - conf = config.read_config(conf_file) + conf = config.read_config(conf_dir + 'service-installer.conf') cmds = conf['ZEPPELIN']['install-commands'] cmds = json.loads(conf['ZEPPELIN']['install-commands']) sh = Shell() +# print(sh.run('pwd')[0]) version = sh.run(cmds[0]) - - fixed_cmd = cmds[1].replace('$VERSION', str(version)) - copy = sh.run(fixed_ver) - +# print("HDP-VERSION: " + str(version[0])) + fixed_cmd = cmds[1].replace('$VERSION', str(version[0])).replace('\n', '') +# print('FIXED COPY COMMAND: ' + fixed_cmd) + copy = sh.run(fixed_cmd) +# print("COPY OUTPUT: " + copy[0]) restart = sh.run(cmds[2]) +# print("Restart output: " + restart[0]) + print("Please open the Ambari Interface and manually deploy the Zeppelin Service.") + raw_input("Press enter twice to continue...") + raw_input("Press enter once to continue...") + +# We've copied the necessary files. Once that completes we need to add it to Ambari + + print('Checking to make sure service is installed') + ambari = config.read_config(conf_dir + 'global-config.conf')['AMBARI'] + installed = check_ambari_service_installed('ZEPPELIN', ambari) + cont = '' + if not installed: + print('Unable to contact Ambari Server. Unsure whether or not Zeppelin was installed') + while not (cont == 'y' or cont == 'n'): + cont = raw_input('Continue attempt to set up Zeppelin for demo?(y/n)') + if not (cont == 'y' or cont == 'n'): + print('Please enter "y" or "n"') + else: + cont = 'y' + + if cont == 'n': + return False + elif cont == 'y': + return True +def check_ambari_service_installed(service_name, ambari_config): + + curl = CurlClient(username=ambari_config['username'], password=ambari_config['password'], port=ambari_config['port'], server=ambari_config['server'], proto=ambari_config['proto']) + + cluster_name = ambari_config['cluster_name'] + request = '/api/v1/clusters/' + cluster_name + '/services/' + service_name + attempts = 0 + while attempts < 10: + output = curl.make_request('GET', request, '-i') + if '200 OK' in output[0]: + print('Service Installed Sucessfully') + return True + else: + attempts += 1 + raw_input('Could not connect.' + str(10-attempts) + ' remaining. Press any key to continue') + + return False diff --git a/tests/test_service_installer.py b/tests/test_service_installer.py index 5e16eec..39034c3 100644 --- a/tests/test_service_installer.py +++ b/tests/test_service_installer.py @@ -1,4 +1,4 @@ -import unittest, json, mock +import unittest, json, mock, scripts from env import scripts from mock import Mock from scripts import service_installer @@ -83,7 +83,7 @@ def test_hdp_select_other_linux(self, mock, mock2): except EnvironmentError as e: assert str(e.message) == 'Must be using one of: CentOS 6.x, CentOS 7.x, Ubuntu 12.x, Ubuntu 14.x' -class TestHDPSelectCheck(unittest.TestCase): +class TestComponentCheck(unittest.TestCase): @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/hdp-select', '']) def test_hdp_select_good(self, mock): @@ -92,9 +92,76 @@ def test_hdp_select_good(self, mock): @mock.patch('scripts.shell.Shell.run', return_value=['', '']) def test_hdp_select_bad(self, mock): assert service_installer.is_hdp_select_installed() == False + + @mock.patch('scripts.shell.Shell.run', return_value=['/usr/bin/ambari-server', '']) + def test_ambari_good(self, mock): + assert service_installer.is_ambari_installed() == True + + @mock.patch('scripts.shell.Shell.run', return_value=['', '']) + def test_ambari_bad(self, mock): + assert service_installer.is_ambari_installed() == False + +class TestZeppelinInstall(unittest.TestCase): - + @mock.patch('scripts.service_installer.is_ambari_installed', return_value=False) + def test_zeppelin_ambari_bad(self, mock): + try: + service_installer.install_zeppelin('../conf/service-installer.conf') + self.fail('Cannot continue installation without Ambari') + except EnvironmentError as e: + assert str(e.message) == 'You must install the demo on the same node as the Ambari server. Install Ambari here or move to another node with Ambari installed before continuing' + + @mock.patch('scripts.service_installer.is_ambari_installed', return_value=True) + @mock.patch('scripts.service_installer.is_hdp_select_installed', return_value=False) + @mock.patch('scripts.service_installer.install_hdp_select', return_value=False) + def test_zeppelin_ambari_good(self, mock, mock2, mock3): #Also HDP select bad + try: + service_installer.install_zeppelin('../conf/service-installer.conf') + self.fail('Cannot continue installation without hdp-select') + except EnvironmentError as e: + assert str(e.message) == 'hdp-select could not be installed. Please install it manually and then re-run the setup.' + + @mock.patch('scripts.service_installer.is_ambari_installed', return_value=True) + @mock.patch('scripts.service_installer.is_hdp_select_installed', return_value=True) + @mock.patch('scripts.service_installer.install_hdp_select', return_value=True) + @mock.patch('scripts.service_installer.check_ambari_service_installed', return_value=True) + @mock.patch('__builtin__.raw_input', return_value='y') + def test_zeppelin_check_is_good(self, mock, mock2, mock3, mock4, mock5): + assert service_installer.install_zeppelin('../conf') == True + + @mock.patch('scripts.service_installer.is_ambari_installed', return_value=True) + @mock.patch('scripts.service_installer.is_hdp_select_installed', return_value=True) + @mock.patch('scripts.service_installer.install_hdp_select', return_value=True) + @mock.patch('scripts.service_installer.check_ambari_service_installed', return_value=False) + @mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'y']) + def test_zeppelin_no_ambari_contact_continue(self, mock, mock2, mock3, mock4, mock5): + assert service_installer.install_zeppelin('../conf') == True + + + @mock.patch('scripts.service_installer.is_ambari_installed', return_value=True) + @mock.patch('scripts.service_installer.is_hdp_select_installed', return_value=True) + @mock.patch('scripts.service_installer.install_hdp_select', return_value=True) + @mock.patch('scripts.service_installer.check_ambari_service_installed', return_value=False) + @mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'n']) + def test_zeppelin_no_ambari_contact_no_continue(self, mock, mock2, mock3, mock4, mock5): + assert service_installer.install_zeppelin('../conf') == False + +class TestAmbariServiceCheck(unittest.TestCase): + + + @mock.patch('scripts.curl_client.CurlClient.make_request', side_effect=[['', ''], ['200 OK', '']]) + @mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'n']) + def test_ambari_check_good(self, mock, mock2): + conf = scripts.config.read_config('../conf/global-config.conf')['AMBARI'] + assert service_installer.check_ambari_service_installed('ZEPPELIN', conf) == True + + @mock.patch('scripts.curl_client.CurlClient.make_request', side_effect=[['200 OK', ''], ['200 OK', '']]) + @mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'n']) + def test_ambari_check_good(self, mock, mock2): + conf = scripts.config.read_config('../conf/global-config.conf')['AMBARI'] + assert service_installer.check_ambari_service_installed('ZEPPELIN', conf) == True + From 51ab552ab302bbb6849b089bfa90a4fcef77a104 Mon Sep 17 00:00:00 2001 From: zblanco Date: Thu, 23 Jun 2016 09:07:31 -0700 Subject: [PATCH 7/7] Last bits for coverage --- tests/test_service_installer.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_service_installer.py b/tests/test_service_installer.py index 39034c3..f99455d 100644 --- a/tests/test_service_installer.py +++ b/tests/test_service_installer.py @@ -158,9 +158,16 @@ def test_ambari_check_good(self, mock, mock2): @mock.patch('scripts.curl_client.CurlClient.make_request', side_effect=[['200 OK', ''], ['200 OK', '']]) @mock.patch('__builtin__.raw_input', side_effect=['\n', '\n', 'v', 'n']) - def test_ambari_check_good(self, mock, mock2): + def test_ambari_check_false(self, mock, mock2): conf = scripts.config.read_config('../conf/global-config.conf')['AMBARI'] assert service_installer.check_ambari_service_installed('ZEPPELIN', conf) == True + + + @mock.patch('scripts.curl_client.CurlClient.make_request', side_effect=[['', ''],['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', ''], ['', '']]) + @mock.patch('__builtin__.raw_input', side_effect=['', '', '', '', '', '', '', '', '', '', '']) + def test_ambari_check_many_attempts(self, mock, mock2): + conf = scripts.config.read_config('../conf/global-config.conf')['AMBARI'] + assert service_installer.check_ambari_service_installed('ZEPPELIN', conf) == False