From 2951b6fb654b94f51012ae017254400ff8fc7aba Mon Sep 17 00:00:00 2001 From: Mingjie Tang Date: Fri, 21 Oct 2016 14:21:39 -0700 Subject: [PATCH 01/12] add livy2 service to spark2 --- .../2.0.0/package/scripts/livy2_server.py | 150 ++++++++++++++++++ .../2.0.0/package/scripts/livy2_service.py | 47 ++++++ .../SPARK2/2.0.0/package/scripts/params.py | 53 ++++++- .../2.0.0/package/scripts/service_check.py | 31 +++- .../2.0.0/package/scripts/setup_livy2.py | 80 ++++++++++ .../2.0.0/package/scripts/status_params.py | 6 + .../SPARK2/configuration/livy2-conf.xml | 66 ++++++++ .../SPARK2/configuration/livy2-env.xml | 95 +++++++++++ .../configuration/livy2-log4j-properties.xml | 42 +++++ .../configuration/livy2-spark-blacklist.xml | 41 +++++ .../HDP/2.5/services/SPARK2/kerberos.json | 121 ++++++++++++++ .../HDP/2.5/services/SPARK2/metainfo.xml | 56 +++++++ .../stacks/HDP/2.5/services/stack_advisor.py | 2 +- .../stacks/2.5/SPARK2/test_spark_livy2.py | 120 ++++++++++++++ 14 files changed, 901 insertions(+), 9 deletions(-) create mode 100644 ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py create mode 100644 ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py create mode 100644 ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py create mode 100644 ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-conf.xml create mode 100644 ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-env.xml create mode 100644 ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-log4j-properties.xml create mode 100644 ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-spark-blacklist.xml create mode 100644 ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/kerberos.json create mode 100644 ambari-server/src/test/python/stacks/2.5/SPARK2/test_spark_livy2.py diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py new file mode 100644 index 00000000000..64d4e4fd9da --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py @@ -0,0 +1,150 @@ +#!/usr/bin/python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.constants import StackFeature +from resource_management.core.exceptions import Fail +from resource_management.core.resources.system import Execute +from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil +from resource_management.libraries.providers.hdfs_resource import HdfsResourceProvider +from resource_management import is_empty +from resource_management import shell +from resource_management.libraries.functions.decorator import retry +from resource_management.core.logger import Logger +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions import conf_select, stack_select + +from livy2_service import livy2_service +from setup_livy2 import setup_livy + +class LivyServer(Script): + + def install(self, env): + import params + env.set_params(params) + + self.install_packages(env) + + def configure(self, env, upgrade_type=None): + import params + env.set_params(params) + + setup_livy(env, 'server', upgrade_type=upgrade_type, action = 'config') + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + + if params.has_ats and params.has_livyserver: + Logger.info("Verifying DFS directories where ATS stores time line data for active and completed applications.") + self.wait_for_dfs_directories_created([params.entity_groupfs_store_dir, params.entity_groupfs_active_dir]) + + self.configure(env) + livy2_service('server', upgrade_type=upgrade_type, action='start') + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + livy2_service('server', upgrade_type=upgrade_type, action='stop') + + def status(self, env): + import status_params + env.set_params(status_params) + + check_process_status(status_params.livy2_server_pid_file) + + # TODO move out and compose with similar method in resourcemanager.py + def wait_for_dfs_directories_created(self, dirs): + import params + + ignored_dfs_dirs = HdfsResourceProvider.get_ignored_resources_list(params.hdfs_resource_ignore_file) + + if params.security_enabled: + Execute(format("{kinit_path_local} -kt {livy2_kerberos_keytab} {livy2_principal}"), + user=params.livy2_user + ) + Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"), + user=params.hdfs_user + ) + + for dir_path in dirs: + self.wait_for_dfs_directory_created(dir_path, ignored_dfs_dirs) + + def get_pid_files(self): + import status_params + return [status_params.livy2_server_pid_file] + + + @retry(times=8, sleep_time=20, backoff_factor=1, err_class=Fail) + def wait_for_dfs_directory_created(self, dir_path, ignored_dfs_dirs): + import params + + + if not is_empty(dir_path): + dir_path = HdfsResourceProvider.parse_path(dir_path) + + if dir_path in ignored_dfs_dirs: + Logger.info("Skipping DFS directory '" + dir_path + "' as it's marked to be ignored.") + return + + Logger.info("Verifying if DFS directory '" + dir_path + "' exists.") + + dir_exists = None + + if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs): + # check with webhdfs is much faster than executing hdfs dfs -test + util = WebHDFSUtil(params.hdfs_site, params.hdfs_user, params.security_enabled) + list_status = util.run_command(dir_path, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False) + dir_exists = ('FileStatus' in list_status) + else: + # have to do time expensive hdfs dfs -d check. + dfs_ret_code = shell.call(format("hdfs --config {hadoop_conf_dir} dfs -test -d " + dir_path), user=params.livy_user)[0] + dir_exists = not dfs_ret_code #dfs -test -d returns 0 in case the dir exists + + if not dir_exists: + raise Fail("DFS directory '" + dir_path + "' does not exist !") + else: + Logger.info("DFS directory '" + dir_path + "' exists.") + + def get_component_name(self): + return "livy2-server" + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + + env.set_params(params) + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + Logger.info("Executing Livy2 Server Stack Upgrade pre-restart") + conf_select.select(params.stack_name, "spark2", params.version) + stack_select.select("livy2-server", params.version) + + def get_log_folder(self): + import params + return params.livy2_log_dir + + def get_user(self): + import params + return params.livy2_user +if __name__ == "__main__": + LivyServer().execute() + diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py new file mode 100644 index 00000000000..e90f6744a51 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +from resource_management.libraries.functions import format +from resource_management.core.resources.system import File, Execute +import threading + +def livy2_service(name, upgrade_type=None, action=None): + import params + + if action == 'start': + livyserver_no_op_test = format( + 'ls {livy2_server_pid_file} >/dev/null 2>&1 && ps -p `cat {livy2_server_pid_file}` >/dev/null 2>&1') + Execute(format('{livy2_server_start}'), + user=params.livy2_user, + environment={'JAVA_HOME': params.java_home}, + not_if=livyserver_no_op_test) + + elif action == 'stop': + Execute(format('{livy2_server_stop}'), + user=params.livy2_user, + environment={'JAVA_HOME': params.java_home} + ) + File(params.livy2_server_pid_file, + action="delete" + ) + + + + diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py index e49756d6cb9..bb5925e1655 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py @@ -38,7 +38,10 @@ SERVER_ROLE_DIRECTORY_MAP = { 'SPARK2_JOBHISTORYSERVER' : 'spark2-historyserver', 'SPARK2_CLIENT' : 'spark2-client', - 'SPARK2_THRIFTSERVER' : 'spark2-thriftserver' + 'SPARK2_THRIFTSERVER' : 'spark2-thriftserver', + 'LIVY2_SERVER' : 'livy2-server', + 'LIVY2_CLIENT' : 'livy2-client' + } component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "SPARK2_CLIENT") @@ -198,4 +201,50 @@ default_fs = default_fs, immutable_paths = get_not_managed_resources(), dfs_type = dfs_type - ) +) + + +# livy related config + +# livy is only supported from HDP 2.5 +has_livyserver = False + +if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stack_version_formatted): + livy2_component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "LIVY2_SERVER") + livy2_conf = format("{stack_root}/current/{livy2_component_directory}/conf") + livy2_log_dir = config['configurations']['livy2-env']['livy2_log_dir'] + livy2_pid_dir = status_params.livy2_pid_dir + livy2_home = format("{stack_root}/current/{livy2_component_directory}") + livy2_user = status_params.livy2_user + livy2_group = status_params.livy2_group + user_group = status_params.user_group + livy2_hdfs_user_dir = format("/user/{livy2_user}") + livy2_server_pid_file = status_params.livy2_server_pid_file + + livy2_server_start = format("{livy2_home}/bin/livy2-server start") + livy2_server_stop = format("{livy2_home}/bin/livy2-server stop") + livy2_logs_dir = format("{livy2_home}/logs") + + livy2_env_sh = config['configurations']['livy2-env']['content'] + livy2_log4j_properties = config['configurations']['livy2-log4j-properties']['content'] + livy2_spark_blacklist_properties = config['configurations']['livy2-spark-blacklist']['content'] + + livy2_kerberos_keytab = config['configurations']['livy2-conf']['livy2.server.kerberos.keytab'] + livy2_kerberos_principal = config['configurations']['livy2-conf']['livy2.server.kerberos.principal'] + + livy2_livyserver_hosts = default("/clusterHostInfo/livy2_server_hosts", []) + + # ats 1.5 properties + entity_groupfs_active_dir = config['configurations']['yarn-site']['yarn.timeline-service.entity-group-fs-store.active-dir'] + entity_groupfs_active_dir_mode = 01777 + entity_groupfs_store_dir = config['configurations']['yarn-site']['yarn.timeline-service.entity-group-fs-store.done-dir'] + entity_groupfs_store_dir_mode = 0700 + is_webhdfs_enabled = hdfs_site['dfs.webhdfs.enabled'] + + if len(livy2_livyserver_hosts) > 0: + has_livyserver = True + if security_enabled: + livy2_principal = livy2_kerberos_principal.replace('_HOST', config['hostname'].lower()) + + livy2_livyserver_port = default('configurations/livy2-conf/livy2.server.port',8998) + diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/service_check.py index 4d49e75ddad..a27eda23cbf 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/service_check.py @@ -31,13 +31,32 @@ def service_check(self, env): if params.security_enabled: spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ") Execute(spark_kinit_cmd, user=params.spark_user) + if (params.has_livyserver): + livy_kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}; ") + Execute(livy_kinit_cmd, user=params.livy2_user) - scheme = "https" if params.ui_ssl_enabled else "http" - Execute(format("curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k {scheme}://{spark_history_server_host}:{spark_history_ui_port} | grep 200"), - tries = 10, - try_sleep=3, - logoutput=True - ) + Execute(format("curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k http://{spark_history_server_host}:{spark_history_ui_port} | grep 200"), + tries=5, + try_sleep=3, + logoutput=True + ) + if params.has_livyserver: + live_livyserver_host = ""; + for livyserver_host in params.livy2_livyserver_hosts: + try: + Execute(format("curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k http://{livyserver_host}:{livy2_livyserver_port}/sessions | grep 200"), + tries=3, + try_sleep=1, + logoutput=True, + user=params.livy2_user + ) + live_livyserver_host = livyserver_host + break + except: + pass + if len(params.livy2_livyserver_hosts) > 0 and live_livyserver_host == "": + raise Fail(format("Connection to all Livy servers failed")) if __name__ == "__main__": SparkServiceCheck().execute() + diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py new file mode 100644 index 00000000000..569e2e2fdf8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py @@ -0,0 +1,80 @@ +#!/usr/bin/python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os +from resource_management import Directory, File, PropertiesFile, InlineTemplate, format + + +def setup_livy(env, type, upgrade_type = None, action = None): + import params + + Directory([params.livy2_pid_dir, params.livy2_log_dir], + owner=params.livy2_user, + group=params.user_group, + mode=0775, + create_parents = True + ) + if type == 'server' and action == 'config': + params.HdfsResource(params.livy2_hdfs_user_dir, + type="directory", + action="create_on_execute", + owner=params.livy2_user, + mode=0775 + ) + params.HdfsResource(None, action="execute") + + # create livy-env.sh in etc/conf dir + File(os.path.join(params.livy2_conf, 'livy2-env.sh'), + owner=params.livy2_user, + group=params.livy2_group, + content=InlineTemplate(params.livy2_env_sh), + mode=0644, + ) + + # create livy.conf in etc/conf dir + PropertiesFile(format("{livy2_conf}/livy2.conf"), + properties = params.config['configurations']['livy2-conf'], + key_value_delimiter = " ", + owner=params.livy2_user, + group=params.livy2_group, + ) + + # create log4j.properties in etc/conf dir + File(os.path.join(params.livy2_conf, 'log4j.properties'), + owner=params.livy2_user, + group=params.livy2_group, + content=params.livy2_log4j_properties, + mode=0644, + ) + + # create spark-blacklist.properties in etc/conf dir + File(os.path.join(params.livy2_conf, 'spark-blacklist.conf'), + owner=params.livy2_user, + group=params.livy2_group, + content=params.livy2_spark_blacklist_properties, + mode=0644, + ) + + Directory(params.livy2_logs_dir, + owner=params.livy2_user, + group=params.livy2_group, + mode=0755, + ) + diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/status_params.py index 2b6aafa2d03..b3669bc3692 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/status_params.py @@ -37,3 +37,9 @@ spark_history_server_pid_file = format("{spark_pid_dir}/spark-{spark_user}-org.apache.spark.deploy.history.HistoryServer-1.pid") spark_thrift_server_pid_file = format("{spark_pid_dir}/spark-{hive_user}-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1.pid") stack_name = default("/hostLevelParams/stack_name", None) + +if "livy2-env" in config['configurations']: + livy2_user = config['configurations']['livy2-env']['livy2_user'] + livy2_group = config['configurations']['livy2-env']['livy2_group'] + livy2_pid_dir = config['configurations']['livy2-env']['livy2_pid_dir'] + livy2_server_pid_file = format("{livy2_pid_dir}/livy2-{livy2_user}-server.pid") \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-conf.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-conf.xml new file mode 100644 index 00000000000..8fa962291c8 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-conf.xml @@ -0,0 +1,66 @@ + + + + + livy2.environment + production + + Specifies Livy's environment. May either be "production" or "development". In "development" + mode, Livy will enable debugging options, such as reporting possible routes on a 404. + defaults to development + + + + + livy2.server.port + 8999 + + What port to start the server on. Defaults to 8998. + + + + + livy2.server.session.timeout + 3600000 + + Time in milliseconds on how long Livy will wait before timing out an idle session. + Default is one hour. + + + + + livy2.impersonation.enabled + true + + If livy should use proxy users when submitting a job. + + + + + livy2.server.csrf_protection.enabled + true + + Whether to enable csrf protection for livy's rest api. + + + + + diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-env.xml new file mode 100644 index 00000000000..686be811bb7 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-env.xml @@ -0,0 +1,95 @@ + + + + + + livy2_user + Livy User + livy2 + USER + + user + false + + + + + livy2_group + Livy Group + livy2 + GROUP + livy group + + user + + + + + livy2_log_dir + /var/log/livy2 + Livy Log Dir + + directory + + + + + livy2_pid_dir + /var/run/livy2 + + directory + + + + + spark_home + /usr/hdp/current/spark2-client + + directory + + + + + + content + This is the jinja template for livy-env.sh file + + #!/usr/bin/env bash + + # - SPARK_HOME Spark which you would like to use in livy + # - HADOOP_CONF_DIR Directory containing the Hadoop / YARN configuration to use. + # - LIVY2_LOG_DIR Where log files are stored. (Default: ${LIVY_HOME}/logs) + # - LIVY2_PID_DIR Where the pid file is stored. (Default: /tmp) + # - LIVY2_SERVER_JAVA_OPTS Java Opts for running livy server (You can set jvm related setting here, like jvm memory/gc algorithm and etc.) + export SPARK_HOME=/usr/hdp/current/spark2-client + export JAVA_HOME={{java_home}} + export HADOOP_CONF_DIR=/etc/hadoop/conf + export LIVY2_LOG_DIR={{livy2_log_dir}} + export LIVY2_PID_DIR={{livy2_pid_dir}} + export LIVY2_SERVER_JAVA_OPTS="-Xmx2g" + + + content + + + + + diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-log4j-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-log4j-properties.xml new file mode 100644 index 00000000000..64e3c0ca34c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-log4j-properties.xml @@ -0,0 +1,42 @@ + + + + + content + Livy2-log4j-Properties + + # Set everything to be logged to the console + log4j.rootCategory=INFO, console + log4j.appender.console=org.apache.log4j.ConsoleAppender + log4j.appender.console.target=System.err + log4j.appender.console.layout=org.apache.log4j.PatternLayout + log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n + + log4j.logger.org.eclipse.jetty=WARN + + + content + false + + + + + diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-spark-blacklist.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-spark-blacklist.xml new file mode 100644 index 00000000000..1cfc64c5a94 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-spark-blacklist.xml @@ -0,0 +1,41 @@ + + + + + content + spark2-blacklist.properties + + # + # Configuration override / blacklist. Defines a list of properties that users are not allowed + # to override when starting Spark sessions. + # + # This file takes a list of property names (one per line). Empty lines and lines starting with "#" + # are ignored. + # + + + content + false + + + + + diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/kerberos.json new file mode 100644 index 00000000000..95814e7388a --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/kerberos.json @@ -0,0 +1,121 @@ +{ + "services": [ + { + "name": "SPARK", + "identities": [ + { + "name": "/smokeuser" + }, + { + "name": "spark2user", + "principal": { + "value": "${spark2-env/spark_user}-${cluster_name|toLower()}@${realm}", + "type" : "user", + "configuration": "spark2-defaults/spark.history.kerberos.principal", + "local_username" : "${spark2-env/spark_user}" + }, + "keytab": { + "file": "${keytab_dir}/spark.headless.keytab", + "owner": { + "name": "${spark2-env/spark_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "spark2-defaults/spark.history.kerberos.keytab" + } + }, + { + "name": "livy2user", + "principal": { + "value": "${livy2-env/livy2_user}/_HOST@${realm}", + "type" : "service", + "configuration": "livy2-conf/livy2.server.kerberos.principal", + "local_username": "${livy2-env/livy2_user}" + }, + "keytab": { + "file": "${keytab_dir}/livy2.service.keytab", + "owner": { + "name": "${livy2-env/livy2_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "livy2-conf/livy2.server.kerberos.keytab" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "livy2-conf/livy2.server.auth.kerberos.principal" + }, + "keytab": { + "configuration": "livy2-conf/livy2.server.auth.kerberos.keytab" + } + } + ], + "configurations": [ + { + "spark-defaults": { + "spark.history.kerberos.enabled": "true" + } + }, + { + "spark-thrift-sparkconf": { + "spark.yarn.keytab": "${spark2-env/hive_kerberos_keytab}", + "spark.yarn.principal": "${spark2-env/hive_kerberos_principal}" + } + }, + { + "livy2-conf": { + "livy2.server.auth.type": "kerberos", + "livy2.impersonation.enabled": "true", + "livy2.superusers": "zeppelin-${cluster_name}" + } + }, + { + "core-site": { + "hadoop.proxyuser.${livy2-env/livy2_user}.groups": "*", + "hadoop.proxyuser.${livy2-env/livy2_user}.hosts": "*" + } + } + ], + "components": [ + { + "name": "SPARK2_JOBHISTORYSERVER", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + } + ] + }, + { + "name": "SPARK2_CLIENT" + }, + { + "name": "SPARK2_THRIFTSERVER", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + }, + { + "name": "/HIVE/HIVE_SERVER/hive_server_hive" + } + ] + }, + { + "name": "LIVY2_SERVER", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/metainfo.xml index 118e8c02025..f69f376f5f2 100755 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/metainfo.xml @@ -24,7 +24,63 @@ SPARK2 2.0.x.2.5 + + + + + LIVY2_SERVER + Livy2 Server + SLAVE + 0+ + true + + + SPARK2/SPARK2_CLIENT + host + + true + + + + HDFS/HDFS_CLIENT + host + + true + + + + YARN/YARN_CLIENT + host + + true + + + + + + PYTHON + 600 + + + + livy2_server + true + + + + + common-services/SPARK2/2.0.0 + + + livy2-conf + livy2-env + livy2-log4j-properties + livy2-spark-blacklist + + + + diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index d2cc4299a35..57715bb4878 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -1908,4 +1908,4 @@ def __getHostsForComponent(self, services, serviceName, componentName): return hosts_for_component def isComponentUsingCardinalityForLayout(self, componentName): - return super(HDP25StackAdvisor, self).isComponentUsingCardinalityForLayout (componentName) or componentName in ['SPARK2_THRIFTSERVER', 'LIVY_SERVER'] + return super(HDP25StackAdvisor, self).isComponentUsingCardinalityForLayout (componentName) or componentName in ['SPARK2_THRIFTSERVER', 'LIVY2_SERVER', 'LIVY_SERVER'] diff --git a/ambari-server/src/test/python/stacks/2.5/SPARK2/test_spark_livy2.py b/ambari-server/src/test/python/stacks/2.5/SPARK2/test_spark_livy2.py new file mode 100644 index 00000000000..cb37f00f918 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.5/SPARK2/test_spark_livy2.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +import json +from mock.mock import MagicMock, patch +from stacks.utils.RMFTestCase import * + +from only_for_platform import not_for_platform, PLATFORM_WINDOWS + +@not_for_platform(PLATFORM_WINDOWS) +@patch("resource_management.libraries.functions.get_stack_version", new=MagicMock(return_value="2.5.0.0-1597")) +class TestSparkClient(RMFTestCase): + COMMON_SERVICES_PACKAGE_DIR = "SPARK/1.2.1/package" + STACK_VERSION = "2.5" + DEFAULT_IMMUTABLE_PATHS = ['/apps/hive/warehouse', '/apps/falcon', '/mr-history/done', '/app-logs', '/tmp'] + def test_configure_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/livy2_server.py", + classname = "LivyServer", + command = "start", + config_file="default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_start_default() + self.assertNoMoreResources() + + def assert_start_default(self): + self.assertResourceCalled('Directory', '/var/run/livy2', + owner = 'livy2', + group = 'hadoop', + create_parents = True, + mode = 0775 + ) + self.assertResourceCalled('Directory', '/var/log/livy2', + owner = 'livy2', + group = 'hadoop', + create_parents = True, + mode = 0775 + ) + self.assertResourceCalled('HdfsResource', '/user/livy2', + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = {u'a': u'b'}, + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + owner = 'livy2', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + type = 'directory', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + dfs_type = '', + mode = 0775, + ) + self.assertResourceCalled('HdfsResource', None, + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = {u'a': u'b'}, + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + dfs_type = '', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + ) + self.assertResourceCalled('File', '/usr/hdp/current/livy2-server/conf/livy2-env.sh', + content = InlineTemplate(self.getConfig()['configurations']['livy2-env']['content']), + owner = 'livy2', + group = 'livy2', + mode = 0644, + ) + self.assertResourceCalled('PropertiesFile', '/usr/hdp/current/livy2-server/conf/livy2.conf', + owner = 'livy2', + key_value_delimiter = ' ', + group = 'livy2', + properties = self.getConfig()['configurations']['livy2-conf'], + ) + self.assertResourceCalled('File', '/usr/hdp/current/livy2-server/conf/log4j.properties', + content = '\n # Set everything to be logged to the console\n log4j.rootCategory=INFO, console\n log4j.appender.console=org.apache.log4j.ConsoleAppender\n log4j.appender.console.target=System.err\n log4j.appender.console.layout=org.apache.log4j.PatternLayout\n log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n log4j.logger.org.eclipse.jetty=WARN', + owner = 'livy2', + group = 'livy2', + mode = 0644, + ) + self.assertResourceCalled('File', '/usr/hdp/current/livy2-server/conf/spark-blacklist.conf', + content = self.getConfig()['configurations']['livy2-spark-blacklist']['content'], + owner = 'livy2', + group = 'livy2', + mode = 0644, + ) + self.assertResourceCalled('Directory', '/usr/hdp/current/livy2-server/logs', + owner = 'livy2', + group = 'livy2', + mode = 0755, + ) + self.assertResourceCalled('Execute', '/usr/hdp/current/livy2-server/bin/livy2-server start', + environment = {'JAVA_HOME': '/usr/jdk64/jdk1.7.0_45'}, + not_if = 'ls /var/run/livy2/livy2-livy-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/livy2/livy2-livy-server.pid` >/dev/null 2>&1', + user = 'livy2' + ) From ce2ef4109d86d06bc0254b3bc3684a6c9927a44d Mon Sep 17 00:00:00 2001 From: Mingjie Tang Date: Tue, 25 Oct 2016 20:54:00 -0700 Subject: [PATCH 02/12] updated from jeff's comment --- .../SPARK2/2.0.0/package/scripts/params.py | 2 +- .../2.5/services/SPARK2/configuration/livy2-conf.xml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py index bb5925e1655..31a4889b089 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py @@ -209,7 +209,7 @@ # livy is only supported from HDP 2.5 has_livyserver = False -if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stack_version_formatted): +if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY2, stack_version_formatted): livy2_component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "LIVY2_SERVER") livy2_conf = format("{stack_root}/current/{livy2_component_directory}/conf") livy2_log_dir = config['configurations']['livy2-env']['livy2_log_dir'] diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-conf.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-conf.xml index 8fa962291c8..caeb138d991 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-conf.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-conf.xml @@ -20,7 +20,7 @@ --> - livy2.environment + livy.environment production Specifies Livy's environment. May either be "production" or "development". In "development" @@ -30,7 +30,7 @@ - livy2.server.port + livy.server.port 8999 What port to start the server on. Defaults to 8998. @@ -38,7 +38,7 @@ - livy2.server.session.timeout + livy.server.session.timeout 3600000 Time in milliseconds on how long Livy will wait before timing out an idle session. @@ -47,7 +47,7 @@ - livy2.impersonation.enabled + livy.impersonation.enabled true If livy should use proxy users when submitting a job. @@ -55,7 +55,7 @@ - livy2.server.csrf_protection.enabled + livy.server.csrf_protection.enabled true Whether to enable csrf protection for livy's rest api. From 13bb49d625a37e189c8eaf5d5dd9d514deb6b87b Mon Sep 17 00:00:00 2001 From: Mingjie Tang Date: Tue, 13 Dec 2016 21:12:58 -0800 Subject: [PATCH 03/12] [BUG-66249] add livy2 as slave in Spark2 service --- .../libraries/functions/constants.py | 1 + .../SPARK2/2.0.0/package/scripts/params.py | 46 ++++----- .../HDP/2.0.6/properties/stack_features.json | 5 + .../stacks/HDP/2.5/role_command_order.json | 3 +- .../HDP/2.5/services/SPARK2/metainfo.xml | 56 ----------- .../SPARK2/configuration/livy2-conf.xml | 0 .../SPARK2/configuration/livy2-env.xml | 0 .../configuration/livy2-log4j-properties.xml | 0 .../configuration/livy2-spark-blacklist.xml | 0 .../services/SPARK2/kerberos.json | 0 .../HDP/2.6/services/SPARK2/metainfo.xml | 94 +++++++++++++++++++ 11 files changed, 127 insertions(+), 78 deletions(-) rename ambari-server/src/main/resources/stacks/HDP/{2.5 => 2.6}/services/SPARK2/configuration/livy2-conf.xml (100%) rename ambari-server/src/main/resources/stacks/HDP/{2.5 => 2.6}/services/SPARK2/configuration/livy2-env.xml (100%) rename ambari-server/src/main/resources/stacks/HDP/{2.5 => 2.6}/services/SPARK2/configuration/livy2-log4j-properties.xml (100%) rename ambari-server/src/main/resources/stacks/HDP/{2.5 => 2.6}/services/SPARK2/configuration/livy2-spark-blacklist.xml (100%) rename ambari-server/src/main/resources/stacks/HDP/{2.5 => 2.6}/services/SPARK2/kerberos.json (100%) mode change 100644 => 100755 ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index 1396bd86b5e..8cf2882b83c 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -60,6 +60,7 @@ class StackFeature: SPARK_16PLUS = "spark_16plus" SPARK_THRIFTSERVER = "spark_thriftserver" SPARK_LIVY = "spark_livy" + SPARK_LIVY2 = "spark_livy2" STORM_KERBEROS = "storm_kerberos" STORM_AMS = "storm_ams" CREATE_KAFKA_BROKER_ID = "create_kafka_broker_id" diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py index 31a4889b089..08d0a1fab78 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py @@ -181,32 +181,16 @@ default_fs = config['configurations']['core-site']['fs.defaultFS'] hdfs_site = config['configurations']['hdfs-site'] +hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore" -dfs_type = default("/commandParams/dfs_type", "") - -import functools -#create partial functions with common arguments for every HdfsResource call -#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code -HdfsResource = functools.partial( - HdfsResource, - user=hdfs_user, - hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", - security_enabled = security_enabled, - keytab = hdfs_user_keytab, - kinit_path_local = kinit_path_local, - hadoop_bin_dir = hadoop_bin_dir, - hadoop_conf_dir = hadoop_conf_dir, - principal_name = hdfs_principal_name, - hdfs_site = hdfs_site, - default_fs = default_fs, - immutable_paths = get_not_managed_resources(), - dfs_type = dfs_type -) +ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", [])) +has_ats = len(ats_host) > 0 +dfs_type = default("/commandParams/dfs_type", "") # livy related config -# livy is only supported from HDP 2.5 +# livy for spark2 is only supported from HDP 2.6 has_livyserver = False if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY2, stack_version_formatted): @@ -248,3 +232,23 @@ livy2_livyserver_port = default('configurations/livy2-conf/livy2.server.port',8998) + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file = hdfs_resource_ignore_file, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs, + immutable_paths = get_not_managed_resources(), + dfs_type = dfs_type +) + diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index dbde58b6194..1bb58639459 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -252,6 +252,11 @@ "description": "Livy as slave component of spark", "min_version": "2.5.0.0" }, + { + "name": "spark_livy2", + "description": "Livy as slave component of spark", + "min_version": "2.6.0.0" + }, { "name": "atlas_ranger_plugin_support", "description": "Atlas Ranger plugin support", diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json index f2616b0141b..f278076e2f3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json @@ -11,8 +11,9 @@ "HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START", "WEBHCAT_SERVER-START", "HIVE_SERVER_INTERACTIVE-START"], "RANGER_ADMIN-START": ["ZOOKEEPER_SERVER-START", "INFRA_SOLR-START"], "LIVY_SERVER-START" : ["NAMENODE-START", "DATANODE-START", "APP_TIMELINE_SERVER-START"], + "LIVY2_SERVER-START" : ["NAMENODE-START", "DATANODE-START", "APP_TIMELINE_SERVER-START"], "SPARK_SERVICE_CHECK-SERVICE_CHECK" : ["SPARK_JOBHISTORYSERVER-START", "LIVY_SERVER-START"], - "SPARK2_SERVICE_CHECK-SERVICE_CHECK" : ["SPARK2_JOBHISTORYSERVER-START", "APP_TIMELINE_SERVER-START"], + "SPARK2_SERVICE_CHECK-SERVICE_CHECK" : ["SPARK2_JOBHISTORYSERVER-START", "LIVY2_SERVER-START"], "ATLAS_SERVER-START": ["KAFKA_BROKER-START", "LOGSEARCH_SOLR-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"] }, "_comment" : "Dependencies that are used when GLUSTERFS is not present in cluster", diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/metainfo.xml index f69f376f5f2..118e8c02025 100755 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/metainfo.xml @@ -24,63 +24,7 @@ SPARK2 2.0.x.2.5 - - - - - LIVY2_SERVER - Livy2 Server - SLAVE - 0+ - true - - - SPARK2/SPARK2_CLIENT - host - - true - - - - HDFS/HDFS_CLIENT - host - - true - - - - YARN/YARN_CLIENT - host - - true - - - - - - PYTHON - 600 - - - - livy2_server - true - - - - - common-services/SPARK2/2.0.0 - - - livy2-conf - livy2-env - livy2-log4j-properties - livy2-spark-blacklist - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-conf.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml similarity index 100% rename from ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-conf.xml rename to ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml similarity index 100% rename from ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-env.xml rename to ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-log4j-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-log4j-properties.xml similarity index 100% rename from ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-log4j-properties.xml rename to ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-log4j-properties.xml diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-spark-blacklist.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-spark-blacklist.xml similarity index 100% rename from ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/configuration/livy2-spark-blacklist.xml rename to ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-spark-blacklist.xml diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/kerberos.json similarity index 100% rename from ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK2/kerberos.json rename to ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/kerberos.json diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml old mode 100644 new mode 100755 index 2f83cf44ba3..78d53c817aa --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml @@ -24,6 +24,100 @@ SPARK2 2.0.x.2.5 + + + + + LIVY2_SERVER + Livy2 Server + SLAVE + 0+ + true + + + SPARK2/SPARK2_CLIENT + host + + true + + + + HDFS/HDFS_CLIENT + host + + true + + + + YARN/YARN_CLIENT + host + + true + + + + + + PYTHON + 600 + + + + livy2_server + true + + + + + + common-services/SPARK2/2.0.0 + + + spark2-defaults + spark2-env + spark2-log4j-properties + spark2-metrics-properties + spark2-thrift-sparkconf + spark2-hive-site-override + spark2-thrift-fairscheduler + livy2-conf + livy2-env + livy2-log4j-properties + livy2-spark-blacklist + + + + + redhat7,amazon2015,redhat6,suse11,suse12 + + + spark2_${stack_version} + + + spark2_${stack_version}-python + + + livy2_${stack_version} + + + + + debian7,ubuntu12,ubuntu14 + + + spark2-${stack_version} + + + spark2-${stack_version}-python + + + livy2-${stack_version} + + + + + + From c29c2f03494e266d972471325140015c112d1c8c Mon Sep 17 00:00:00 2001 From: Mingjie Tang Date: Wed, 21 Dec 2016 14:19:47 -0800 Subject: [PATCH 04/12] update the livy2 to livy-spark2 --- .../SPARK2/2.0.0/package/scripts/params.py | 12 ++++++------ .../SPARK2/2.0.0/package/scripts/setup_livy2.py | 4 ++-- .../SPARK2/2.0.0/package/scripts/status_params.py | 2 +- .../2.6/services/SPARK2/configuration/livy2-conf.xml | 2 +- .../2.6/services/SPARK2/configuration/livy2-env.xml | 10 +++++----- .../stacks/HDP/2.6/services/SPARK2/metainfo.xml | 8 ++++---- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py index 08d0a1fab78..2cc1f5562f0 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py @@ -39,8 +39,8 @@ 'SPARK2_JOBHISTORYSERVER' : 'spark2-historyserver', 'SPARK2_CLIENT' : 'spark2-client', 'SPARK2_THRIFTSERVER' : 'spark2-thriftserver', - 'LIVY2_SERVER' : 'livy2-server', - 'LIVY2_CLIENT' : 'livy2-client' + 'LIVY-SPARK2_SERVER' : 'livy-spark2-server', + 'LIVY-SPARK2_CLIENT' : 'livy-spark2-client' } @@ -194,7 +194,7 @@ has_livyserver = False if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY2, stack_version_formatted): - livy2_component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "LIVY2_SERVER") + livy2_component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "LIVY-SPARK2_SERVER") livy2_conf = format("{stack_root}/current/{livy2_component_directory}/conf") livy2_log_dir = config['configurations']['livy2-env']['livy2_log_dir'] livy2_pid_dir = status_params.livy2_pid_dir @@ -205,8 +205,8 @@ livy2_hdfs_user_dir = format("/user/{livy2_user}") livy2_server_pid_file = status_params.livy2_server_pid_file - livy2_server_start = format("{livy2_home}/bin/livy2-server start") - livy2_server_stop = format("{livy2_home}/bin/livy2-server stop") + livy2_server_start = format("{livy2_home}/bin/livy-server start") + livy2_server_stop = format("{livy2_home}/bin/livy-server stop") livy2_logs_dir = format("{livy2_home}/logs") livy2_env_sh = config['configurations']['livy2-env']['content'] @@ -230,7 +230,7 @@ if security_enabled: livy2_principal = livy2_kerberos_principal.replace('_HOST', config['hostname'].lower()) - livy2_livyserver_port = default('configurations/livy2-conf/livy2.server.port',8998) + livy2_livyserver_port = default('configurations/livy2-conf/livy2.server.port',8999) import functools diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py index 569e2e2fdf8..ab44cf8f35d 100644 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py @@ -41,7 +41,7 @@ def setup_livy(env, type, upgrade_type = None, action = None): params.HdfsResource(None, action="execute") # create livy-env.sh in etc/conf dir - File(os.path.join(params.livy2_conf, 'livy2-env.sh'), + File(os.path.join(params.livy2_conf, 'livy-env.sh'), owner=params.livy2_user, group=params.livy2_group, content=InlineTemplate(params.livy2_env_sh), @@ -49,7 +49,7 @@ def setup_livy(env, type, upgrade_type = None, action = None): ) # create livy.conf in etc/conf dir - PropertiesFile(format("{livy2_conf}/livy2.conf"), + PropertiesFile(format("{livy2_conf}/livy.conf"), properties = params.config['configurations']['livy2-conf'], key_value_delimiter = " ", owner=params.livy2_user, diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/status_params.py index b3669bc3692..2d11ac252a7 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/status_params.py @@ -42,4 +42,4 @@ livy2_user = config['configurations']['livy2-env']['livy2_user'] livy2_group = config['configurations']['livy2-env']['livy2_group'] livy2_pid_dir = config['configurations']['livy2-env']['livy2_pid_dir'] - livy2_server_pid_file = format("{livy2_pid_dir}/livy2-{livy2_user}-server.pid") \ No newline at end of file + livy2_server_pid_file = format("{livy2_pid_dir}/livy-{livy2_user}-server.pid") \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml index caeb138d991..6faba8895fa 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml @@ -33,7 +33,7 @@ livy.server.port 8999 - What port to start the server on. Defaults to 8998. + What port to start the server on. Defaults to 8999. diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml index 686be811bb7..62f5cdee4fd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml @@ -44,7 +44,7 @@ livy2_log_dir - /var/log/livy2 + /var/log/livy-spark2 Livy Log Dir directory @@ -53,7 +53,7 @@ livy2_pid_dir - /var/run/livy2 + /var/run/livy-spark2 directory @@ -82,9 +82,9 @@ export SPARK_HOME=/usr/hdp/current/spark2-client export JAVA_HOME={{java_home}} export HADOOP_CONF_DIR=/etc/hadoop/conf - export LIVY2_LOG_DIR={{livy2_log_dir}} - export LIVY2_PID_DIR={{livy2_pid_dir}} - export LIVY2_SERVER_JAVA_OPTS="-Xmx2g" + export LIVY_LOG_DIR={{livy2_log_dir}} + export LIVY_PID_DIR={{livy2_pid_dir}} + export LIVY_SERVER_JAVA_OPTS="-Xmx2g" content diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml index 78d53c817aa..703c196c25c 100755 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml @@ -28,8 +28,8 @@ - LIVY2_SERVER - Livy2 Server + LIVY-SPARK2_SERVER + Livy for Spark2 Server SLAVE 0+ true @@ -97,7 +97,7 @@ spark2_${stack_version}-python - livy2_${stack_version} + livy_${stack_version}-spark2 @@ -111,7 +111,7 @@ spark2-${stack_version}-python - livy2-${stack_version} + livy-${stack_version}-spark2 From 23fdeced3d209818df779d9ac5524e5673205586 Mon Sep 17 00:00:00 2001 From: Mingjie Tang Date: Thu, 22 Dec 2016 12:48:28 -0800 Subject: [PATCH 05/12] update with jeff and bikas' comments for the kerberos issue --- .../SPARK2/2.0.0/package/scripts/params.py | 4 ++-- .../services/SPARK2/configuration/livy2-env.xml | 4 ++-- .../stacks/HDP/2.6/services/SPARK2/kerberos.json | 14 +++++++------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py index 2cc1f5562f0..29e379c7e69 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py @@ -213,8 +213,8 @@ livy2_log4j_properties = config['configurations']['livy2-log4j-properties']['content'] livy2_spark_blacklist_properties = config['configurations']['livy2-spark-blacklist']['content'] - livy2_kerberos_keytab = config['configurations']['livy2-conf']['livy2.server.kerberos.keytab'] - livy2_kerberos_principal = config['configurations']['livy2-conf']['livy2.server.kerberos.principal'] + livy2_kerberos_keytab = config['configurations']['livy2-conf']['livy.server.kerberos.keytab'] + livy2_kerberos_principal = config['configurations']['livy2-conf']['livy.server.kerberos.principal'] livy2_livyserver_hosts = default("/clusterHostInfo/livy2_server_hosts", []) diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml index 62f5cdee4fd..dfcba26d567 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml @@ -23,7 +23,7 @@ livy2_user Livy User - livy2 + livy USER user @@ -34,7 +34,7 @@ livy2_group Livy Group - livy2 + livy GROUP livy group diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/kerberos.json index 95814e7388a..7eb8a5a4d4b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/kerberos.json @@ -32,7 +32,7 @@ "principal": { "value": "${livy2-env/livy2_user}/_HOST@${realm}", "type" : "service", - "configuration": "livy2-conf/livy2.server.kerberos.principal", + "configuration": "livy2-conf/livy.server.kerberos.principal", "local_username": "${livy2-env/livy2_user}" }, "keytab": { @@ -45,16 +45,16 @@ "name": "${cluster-env/user_group}", "access": "" }, - "configuration": "livy2-conf/livy2.server.kerberos.keytab" + "configuration": "livy2-conf/livy.server.kerberos.keytab" } }, { "name": "/spnego", "principal": { - "configuration": "livy2-conf/livy2.server.auth.kerberos.principal" + "configuration": "livy2-conf/livy.server.auth.kerberos.principal" }, "keytab": { - "configuration": "livy2-conf/livy2.server.auth.kerberos.keytab" + "configuration": "livy2-conf/livy.server.auth.kerberos.keytab" } } ], @@ -72,9 +72,9 @@ }, { "livy2-conf": { - "livy2.server.auth.type": "kerberos", - "livy2.impersonation.enabled": "true", - "livy2.superusers": "zeppelin-${cluster_name}" + "livy.server.auth.type": "kerberos", + "livy.impersonation.enabled": "true", + "livy.superusers": "zeppelin-${cluster_name}" } }, { From f2a71a4cc1cb012a398a431d85f72908f37ab98b Mon Sep 17 00:00:00 2001 From: Mingjie Tang Date: Thu, 22 Dec 2016 13:23:32 -0800 Subject: [PATCH 06/12] revert livy-spark2 to livy2 --- .../SPARK2/2.0.0/package/scripts/params.py | 8 ++++---- .../2.6/services/SPARK2/configuration/livy2-env.xml | 10 +++++----- .../stacks/HDP/2.6/services/SPARK2/metainfo.xml | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py index 29e379c7e69..93f78d4f8c1 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py @@ -39,8 +39,8 @@ 'SPARK2_JOBHISTORYSERVER' : 'spark2-historyserver', 'SPARK2_CLIENT' : 'spark2-client', 'SPARK2_THRIFTSERVER' : 'spark2-thriftserver', - 'LIVY-SPARK2_SERVER' : 'livy-spark2-server', - 'LIVY-SPARK2_CLIENT' : 'livy-spark2-client' + 'LIVY2_SERVER' : 'livy2-server', + 'LIVY2_CLIENT' : 'livy2-client' } @@ -194,7 +194,7 @@ has_livyserver = False if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY2, stack_version_formatted): - livy2_component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "LIVY-SPARK2_SERVER") + livy2_component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "LIVY2_SERVER") livy2_conf = format("{stack_root}/current/{livy2_component_directory}/conf") livy2_log_dir = config['configurations']['livy2-env']['livy2_log_dir'] livy2_pid_dir = status_params.livy2_pid_dir @@ -230,7 +230,7 @@ if security_enabled: livy2_principal = livy2_kerberos_principal.replace('_HOST', config['hostname'].lower()) - livy2_livyserver_port = default('configurations/livy2-conf/livy2.server.port',8999) + livy2_livyserver_port = default('configurations/livy2-conf/livy.server.port',8999) import functools diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml index dfcba26d567..71b1f21a9e1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml @@ -44,7 +44,7 @@ livy2_log_dir - /var/log/livy-spark2 + /var/log/livy2 Livy Log Dir directory @@ -53,7 +53,7 @@ livy2_pid_dir - /var/run/livy-spark2 + /var/run/livy2 directory @@ -76,9 +76,9 @@ # - SPARK_HOME Spark which you would like to use in livy # - HADOOP_CONF_DIR Directory containing the Hadoop / YARN configuration to use. - # - LIVY2_LOG_DIR Where log files are stored. (Default: ${LIVY_HOME}/logs) - # - LIVY2_PID_DIR Where the pid file is stored. (Default: /tmp) - # - LIVY2_SERVER_JAVA_OPTS Java Opts for running livy server (You can set jvm related setting here, like jvm memory/gc algorithm and etc.) + # - LIVY_LOG_DIR Where log files are stored. (Default: ${LIVY_HOME}/logs) + # - LIVY_PID_DIR Where the pid file is stored. (Default: /tmp) + # - LIVY_SERVER_JAVA_OPTS Java Opts for running livy server (You can set jvm related setting here, like jvm memory/gc algorithm and etc.) export SPARK_HOME=/usr/hdp/current/spark2-client export JAVA_HOME={{java_home}} export HADOOP_CONF_DIR=/etc/hadoop/conf diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml index 703c196c25c..444343bc88b 100755 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml @@ -28,7 +28,7 @@ - LIVY-SPARK2_SERVER + LIVY2_SERVER Livy for Spark2 Server SLAVE 0+ @@ -97,7 +97,7 @@ spark2_${stack_version}-python - livy_${stack_version}-spark2 + livy2_${stack_version} @@ -111,7 +111,7 @@ spark2-${stack_version}-python - livy-${stack_version}-spark2 + livy2-${stack_version} From d54938e887d39f230189a2cd250ff7f46ffd61d8 Mon Sep 17 00:00:00 2001 From: Mingjie Tang Date: Thu, 22 Dec 2016 14:48:52 -0800 Subject: [PATCH 07/12] Apply AMBARI-19025, AMBARI-18758 and AMBARI-18923 to Livy2 of Spark2 --- .../SPARK2/2.0.0/package/scripts/params.py | 13 +++- .../SPARK2/configuration/livy2-conf.xml | 16 ++++ .../configuration/livy2-spark-blacklist.xml | 15 +++- .../HDP/2.6/services/SPARK2/kerberos.json | 73 ++++++++++--------- 4 files changed, 78 insertions(+), 39 deletions(-) diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py index 93f78d4f8c1..99a0441ec68 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py @@ -213,10 +213,17 @@ livy2_log4j_properties = config['configurations']['livy2-log4j-properties']['content'] livy2_spark_blacklist_properties = config['configurations']['livy2-spark-blacklist']['content'] - livy2_kerberos_keytab = config['configurations']['livy2-conf']['livy.server.kerberos.keytab'] - livy2_kerberos_principal = config['configurations']['livy2-conf']['livy.server.kerberos.principal'] + if 'livy.server.kerberos.keytab' in config['configurations']['livy2-conf']: + livy_kerberos_keytab = config['configurations']['livy2-conf']['livy.server.kerberos.keytab'] + else: + livy_kerberos_keytab = config['configurations']['livy2-conf']['livy.server.launch.kerberos.keytab'] + if 'livy.server.kerberos.principal' in config['configurations']['livy2-conf']: + livy_kerberos_principal = config['configurations']['livy2-conf']['livy.server.kerberos.principal'] + else: + livy_kerberos_principal = config['configurations']['livy2-conf']['livy.server.launch.kerberos.principal'] - livy2_livyserver_hosts = default("/clusterHostInfo/livy2_server_hosts", []) + +livy2_livyserver_hosts = default("/clusterHostInfo/livy2_server_hosts", []) # ats 1.5 properties entity_groupfs_active_dir = config['configurations']['yarn-site']['yarn.timeline-service.entity-group-fs-store.active-dir'] diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml index 6faba8895fa..f2a2316b3e7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml @@ -54,6 +54,14 @@ + + livy.impersonation.enabled + true + + If livy should use proxy users when submitting a job. + + + livy.server.csrf_protection.enabled true @@ -62,5 +70,13 @@ + + livy.spark.master + yarn-cluster + + spark.master property for spark engine + + + diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-spark-blacklist.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-spark-blacklist.xml index 1cfc64c5a94..8319b21ffd7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-spark-blacklist.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-spark-blacklist.xml @@ -30,6 +30,18 @@ # This file takes a list of property names (one per line). Empty lines and lines starting with "#" # are ignored. # + + # Disallow overriding the master and the deploy mode. + spark.master + spark.submit.deployMode + + # Disallow overriding the location of Spark cached jars. + spark.yarn.jar + spark.yarn.jars + spark.yarn.archive + + # Don't allow users to override the RSC timeout. + livy.rsc.server.idle_timeout content @@ -37,5 +49,4 @@ - - + \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/kerberos.json index 7eb8a5a4d4b..42626704b8d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/kerberos.json @@ -26,36 +26,6 @@ }, "configuration": "spark2-defaults/spark.history.kerberos.keytab" } - }, - { - "name": "livy2user", - "principal": { - "value": "${livy2-env/livy2_user}/_HOST@${realm}", - "type" : "service", - "configuration": "livy2-conf/livy.server.kerberos.principal", - "local_username": "${livy2-env/livy2_user}" - }, - "keytab": { - "file": "${keytab_dir}/livy2.service.keytab", - "owner": { - "name": "${livy2-env/livy2_user}", - "access": "r" - }, - "group": { - "name": "${cluster-env/user_group}", - "access": "" - }, - "configuration": "livy2-conf/livy.server.kerberos.keytab" - } - }, - { - "name": "/spnego", - "principal": { - "configuration": "livy2-conf/livy.server.auth.kerberos.principal" - }, - "keytab": { - "configuration": "livy2-conf/livy.server.auth.kerberos.keytab" - } } ], "configurations": [ @@ -89,7 +59,8 @@ "name": "SPARK2_JOBHISTORYSERVER", "identities": [ { - "name": "/HDFS/NAMENODE/hdfs" + "name": "hdfs", + "reference": "/HDFS/NAMENODE/hdfs" } ] }, @@ -100,10 +71,12 @@ "name": "SPARK2_THRIFTSERVER", "identities": [ { - "name": "/HDFS/NAMENODE/hdfs" + "name": "hdfs", + "reference": "/HDFS/NAMENODE/hdfs" }, { - "name": "/HIVE/HIVE_SERVER/hive_server_hive" + "name": "hive_server_hive", + "reference": "/HIVE/HIVE_SERVER/hive_server_hive" } ] }, @@ -111,7 +84,39 @@ "name": "LIVY2_SERVER", "identities": [ { - "name": "/HDFS/NAMENODE/hdfs" + "name": "hdfs", + "reference": "/HDFS/NAMENODE/hdfs" + }, + { + "name": "livyuser", + "principal": { + "value": "${livy2-env/livy2_user}/_HOST@${realm}", + "type" : "service", + "configuration": "livy2-conf/livy.server.launch.kerberos.principal", + "local_username": "${livy2-env/livy2_user}" + }, + "keytab": { + "file": "${keytab_dir}/livy.service.keytab", + "owner": { + "name": "${livy2-env/livy2_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "livy2-conf/livy.server.launch.kerberos.keytab" + } + }, + { + "name": "livy_spnego", + "reference": "/spnego", + "principal": { + "configuration": "livy2-conf/livy.server.auth.kerberos.principal" + }, + "keytab": { + "configuration": "livy2-conf/livy.server.auth.kerberos.keytab" + } } ] } From c2913af37ff2090452034d54568d952ac257f05d Mon Sep 17 00:00:00 2001 From: Mingjie Tang Date: Thu, 22 Dec 2016 16:21:28 -0800 Subject: [PATCH 08/12] update the test script for the livy2 --- .../{2.5 => 2.6}/SPARK2/test_spark_livy2.py | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) rename ambari-server/src/test/python/stacks/{2.5 => 2.6}/SPARK2/test_spark_livy2.py (86%) diff --git a/ambari-server/src/test/python/stacks/2.5/SPARK2/test_spark_livy2.py b/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py similarity index 86% rename from ambari-server/src/test/python/stacks/2.5/SPARK2/test_spark_livy2.py rename to ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py index cb37f00f918..3f9ec90c73d 100644 --- a/ambari-server/src/test/python/stacks/2.5/SPARK2/test_spark_livy2.py +++ b/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py @@ -26,8 +26,8 @@ @not_for_platform(PLATFORM_WINDOWS) @patch("resource_management.libraries.functions.get_stack_version", new=MagicMock(return_value="2.5.0.0-1597")) class TestSparkClient(RMFTestCase): - COMMON_SERVICES_PACKAGE_DIR = "SPARK/1.2.1/package" - STACK_VERSION = "2.5" + COMMON_SERVICES_PACKAGE_DIR = "SPARK2/2.0.0/package" + STACK_VERSION = "2.6" DEFAULT_IMMUTABLE_PATHS = ['/apps/hive/warehouse', '/apps/falcon', '/mr-history/done', '/app-logs', '/tmp'] def test_configure_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/livy2_server.py", @@ -42,13 +42,13 @@ def test_configure_default(self): def assert_start_default(self): self.assertResourceCalled('Directory', '/var/run/livy2', - owner = 'livy2', + owner = 'livy', group = 'hadoop', create_parents = True, mode = 0775 ) self.assertResourceCalled('Directory', '/var/log/livy2', - owner = 'livy2', + owner = 'livy', group = 'hadoop', create_parents = True, mode = 0775 @@ -63,7 +63,7 @@ def assert_start_default(self): kinit_path_local = '/usr/bin/kinit', principal_name = UnknownConfigurationMock(), user = 'hdfs', - owner = 'livy2', + owner = 'livy', hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', type = 'directory', action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', @@ -84,37 +84,37 @@ def assert_start_default(self): dfs_type = '', hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', ) - self.assertResourceCalled('File', '/usr/hdp/current/livy2-server/conf/livy2-env.sh', + self.assertResourceCalled('File', '/usr/hdp/current/livy2-server/conf/livy-env.sh', content = InlineTemplate(self.getConfig()['configurations']['livy2-env']['content']), - owner = 'livy2', - group = 'livy2', + owner = 'livy', + group = 'livy', mode = 0644, ) - self.assertResourceCalled('PropertiesFile', '/usr/hdp/current/livy2-server/conf/livy2.conf', - owner = 'livy2', + self.assertResourceCalled('PropertiesFile', '/usr/hdp/current/livy2-server/conf/livy.conf', + owner = 'livy', key_value_delimiter = ' ', - group = 'livy2', + group = 'hadoop', properties = self.getConfig()['configurations']['livy2-conf'], ) self.assertResourceCalled('File', '/usr/hdp/current/livy2-server/conf/log4j.properties', content = '\n # Set everything to be logged to the console\n log4j.rootCategory=INFO, console\n log4j.appender.console=org.apache.log4j.ConsoleAppender\n log4j.appender.console.target=System.err\n log4j.appender.console.layout=org.apache.log4j.PatternLayout\n log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n log4j.logger.org.eclipse.jetty=WARN', - owner = 'livy2', - group = 'livy2', + owner = 'livy', + group = 'livy', mode = 0644, ) self.assertResourceCalled('File', '/usr/hdp/current/livy2-server/conf/spark-blacklist.conf', content = self.getConfig()['configurations']['livy2-spark-blacklist']['content'], - owner = 'livy2', - group = 'livy2', + owner = 'livy', + group = 'livy', mode = 0644, ) self.assertResourceCalled('Directory', '/usr/hdp/current/livy2-server/logs', - owner = 'livy2', - group = 'livy2', + owner = 'livy', + group = 'livy', mode = 0755, ) - self.assertResourceCalled('Execute', '/usr/hdp/current/livy2-server/bin/livy2-server start', + self.assertResourceCalled('Execute', '/usr/hdp/current/livy2-server/bin/livy-server start', environment = {'JAVA_HOME': '/usr/jdk64/jdk1.7.0_45'}, - not_if = 'ls /var/run/livy2/livy2-livy-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/livy2/livy2-livy-server.pid` >/dev/null 2>&1', - user = 'livy2' + not_if = 'ls /var/run/livy2/livy-livy-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/livy2/livy-livy-server.pid` >/dev/null 2>&1', + user = 'livy' ) From 01860f9808a39210d1323fb2f13781bbd49d07d9 Mon Sep 17 00:00:00 2001 From: Mingjie Tang Date: Thu, 29 Dec 2016 14:09:15 -0800 Subject: [PATCH 09/12] test livy2 from clean ambari --- .../common-services/SPARK2/2.0.0/package/scripts/params.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py index 99a0441ec68..d763c94c030 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py @@ -222,8 +222,7 @@ else: livy_kerberos_principal = config['configurations']['livy2-conf']['livy.server.launch.kerberos.principal'] - -livy2_livyserver_hosts = default("/clusterHostInfo/livy2_server_hosts", []) + livy2_livyserver_hosts = default("/clusterHostInfo/livy2_server_hosts", []) # ats 1.5 properties entity_groupfs_active_dir = config['configurations']['yarn-site']['yarn.timeline-service.entity-group-fs-store.active-dir'] From 1199531aa070ad036b5b20c7b7b71dc57e87a867 Mon Sep 17 00:00:00 2001 From: Mingjie Tang Date: Wed, 4 Jan 2017 18:08:12 -0800 Subject: [PATCH 10/12] for security cluster --- .../2.0.0/package/scripts/livy2_server.py | 2 +- .../SPARK2/2.0.0/package/scripts/params.py | 84 +++++++++---------- .../HDP/2.6/services/SPARK2/kerberos.json | 2 +- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py index 64d4e4fd9da..b19909599b1 100644 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py @@ -80,7 +80,7 @@ def wait_for_dfs_directories_created(self, dirs): ignored_dfs_dirs = HdfsResourceProvider.get_ignored_resources_list(params.hdfs_resource_ignore_file) if params.security_enabled: - Execute(format("{kinit_path_local} -kt {livy2_kerberos_keytab} {livy2_principal}"), + Execute(format("{kinit_path_local} -kt {livy_kerberos_keytab} {livy2_principal}"), user=params.livy2_user ) Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"), diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py index d763c94c030..452746dc580 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py @@ -193,50 +193,50 @@ # livy for spark2 is only supported from HDP 2.6 has_livyserver = False -if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY2, stack_version_formatted): - livy2_component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "LIVY2_SERVER") - livy2_conf = format("{stack_root}/current/{livy2_component_directory}/conf") - livy2_log_dir = config['configurations']['livy2-env']['livy2_log_dir'] - livy2_pid_dir = status_params.livy2_pid_dir - livy2_home = format("{stack_root}/current/{livy2_component_directory}") - livy2_user = status_params.livy2_user - livy2_group = status_params.livy2_group - user_group = status_params.user_group - livy2_hdfs_user_dir = format("/user/{livy2_user}") - livy2_server_pid_file = status_params.livy2_server_pid_file - - livy2_server_start = format("{livy2_home}/bin/livy-server start") - livy2_server_stop = format("{livy2_home}/bin/livy-server stop") - livy2_logs_dir = format("{livy2_home}/logs") - - livy2_env_sh = config['configurations']['livy2-env']['content'] - livy2_log4j_properties = config['configurations']['livy2-log4j-properties']['content'] - livy2_spark_blacklist_properties = config['configurations']['livy2-spark-blacklist']['content'] - - if 'livy.server.kerberos.keytab' in config['configurations']['livy2-conf']: - livy_kerberos_keytab = config['configurations']['livy2-conf']['livy.server.kerberos.keytab'] - else: - livy_kerberos_keytab = config['configurations']['livy2-conf']['livy.server.launch.kerberos.keytab'] - if 'livy.server.kerberos.principal' in config['configurations']['livy2-conf']: - livy_kerberos_principal = config['configurations']['livy2-conf']['livy.server.kerberos.principal'] - else: - livy_kerberos_principal = config['configurations']['livy2-conf']['livy.server.launch.kerberos.principal'] - - livy2_livyserver_hosts = default("/clusterHostInfo/livy2_server_hosts", []) +#if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY2, stack_version_formatted): +livy2_component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "LIVY2_SERVER") +livy2_conf = format("{stack_root}/current/{livy2_component_directory}/conf") +livy2_log_dir = config['configurations']['livy2-env']['livy2_log_dir'] +livy2_pid_dir = status_params.livy2_pid_dir +livy2_home = format("{stack_root}/current/{livy2_component_directory}") +livy2_user = status_params.livy2_user +livy2_group = status_params.livy2_group +user_group = status_params.user_group +livy2_hdfs_user_dir = format("/user/{livy2_user}") +livy2_server_pid_file = status_params.livy2_server_pid_file + +livy2_server_start = format("{livy2_home}/bin/livy-server start") +livy2_server_stop = format("{livy2_home}/bin/livy-server stop") +livy2_logs_dir = format("{livy2_home}/logs") + +livy2_env_sh = config['configurations']['livy2-env']['content'] +livy2_log4j_properties = config['configurations']['livy2-log4j-properties']['content'] +livy2_spark_blacklist_properties = config['configurations']['livy2-spark-blacklist']['content'] + +if 'livy.server.kerberos.keytab' in config['configurations']['livy2-conf']: + livy_kerberos_keytab = config['configurations']['livy2-conf']['livy.server.kerberos.keytab'] +else: + livy_kerberos_keytab = config['configurations']['livy2-conf']['livy.server.launch.kerberos.keytab'] +if 'livy.server.kerberos.principal' in config['configurations']['livy2-conf']: + livy_kerberos_principal = config['configurations']['livy2-conf']['livy.server.kerberos.principal'] +else: + livy_kerberos_principal = config['configurations']['livy2-conf']['livy.server.launch.kerberos.principal'] + +livy2_livyserver_hosts = default("/clusterHostInfo/livy2_server_hosts", []) # ats 1.5 properties - entity_groupfs_active_dir = config['configurations']['yarn-site']['yarn.timeline-service.entity-group-fs-store.active-dir'] - entity_groupfs_active_dir_mode = 01777 - entity_groupfs_store_dir = config['configurations']['yarn-site']['yarn.timeline-service.entity-group-fs-store.done-dir'] - entity_groupfs_store_dir_mode = 0700 - is_webhdfs_enabled = hdfs_site['dfs.webhdfs.enabled'] - - if len(livy2_livyserver_hosts) > 0: - has_livyserver = True - if security_enabled: - livy2_principal = livy2_kerberos_principal.replace('_HOST', config['hostname'].lower()) - - livy2_livyserver_port = default('configurations/livy2-conf/livy.server.port',8999) +entity_groupfs_active_dir = config['configurations']['yarn-site']['yarn.timeline-service.entity-group-fs-store.active-dir'] +entity_groupfs_active_dir_mode = 01777 +entity_groupfs_store_dir = config['configurations']['yarn-site']['yarn.timeline-service.entity-group-fs-store.done-dir'] +entity_groupfs_store_dir_mode = 0700 +is_webhdfs_enabled = hdfs_site['dfs.webhdfs.enabled'] + +if len(livy2_livyserver_hosts) > 0: + has_livyserver = True + if security_enabled: + livy2_principal = livy_kerberos_principal.replace('_HOST', config['hostname'].lower()) + +livy2_livyserver_port = default('configurations/livy2-conf/livy.server.port',8999) import functools diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/kerberos.json index 42626704b8d..a16e2e1f949 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/kerberos.json @@ -1,7 +1,7 @@ { "services": [ { - "name": "SPARK", + "name": "SPARK2", "identities": [ { "name": "/smokeuser" From daae0320c88989ea80a4a93ccec2c3f15a77f4c5 Mon Sep 17 00:00:00 2001 From: Mingjie Tang Date: Thu, 5 Jan 2017 15:30:50 -0800 Subject: [PATCH 11/12] fixed based on Fernandez's comments --- .../2.0.0/package/scripts/livy2_server.py | 157 +++++++++--------- .../2.0.0/package/scripts/livy2_service.py | 37 +++-- .../2.0.0/package/scripts/setup_livy2.py | 96 +++++------ .../SPARK2/configuration/livy2-conf.xml | 14 +- .../SPARK2/configuration/livy2-env.xml | 12 +- .../configuration/livy2-log4j-properties.xml | 2 +- .../configuration/livy2-spark-blacklist.xml | 2 +- 7 files changed, 160 insertions(+), 160 deletions(-) diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py index b19909599b1..8c66998d3af 100644 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py @@ -38,113 +38,112 @@ class LivyServer(Script): - def install(self, env): - import params - env.set_params(params) + def install(self, env): + import params + env.set_params(params) - self.install_packages(env) + self.install_packages(env) - def configure(self, env, upgrade_type=None): - import params - env.set_params(params) + def configure(self, env, upgrade_type=None): + import params + env.set_params(params) - setup_livy(env, 'server', upgrade_type=upgrade_type, action = 'config') + setup_livy(env, 'server', upgrade_type=upgrade_type, action = 'config') - def start(self, env, upgrade_type=None): - import params - env.set_params(params) + def start(self, env, upgrade_type=None): + import params + env.set_params(params) - if params.has_ats and params.has_livyserver: - Logger.info("Verifying DFS directories where ATS stores time line data for active and completed applications.") - self.wait_for_dfs_directories_created([params.entity_groupfs_store_dir, params.entity_groupfs_active_dir]) + if params.has_ats and params.has_livyserver: + Logger.info("Verifying DFS directories where ATS stores time line data for active and completed applications.") + self.wait_for_dfs_directories_created([params.entity_groupfs_store_dir, params.entity_groupfs_active_dir]) - self.configure(env) - livy2_service('server', upgrade_type=upgrade_type, action='start') + self.configure(env) + livy2_service('server', upgrade_type=upgrade_type, action='start') - def stop(self, env, upgrade_type=None): - import params - env.set_params(params) + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) - livy2_service('server', upgrade_type=upgrade_type, action='stop') + livy2_service('server', upgrade_type=upgrade_type, action='stop') - def status(self, env): - import status_params - env.set_params(status_params) + def status(self, env): + import status_params + env.set_params(status_params) - check_process_status(status_params.livy2_server_pid_file) + check_process_status(status_params.livy2_server_pid_file) - # TODO move out and compose with similar method in resourcemanager.py - def wait_for_dfs_directories_created(self, dirs): - import params + # TODO move out and compose with similar method in resourcemanager.py + def wait_for_dfs_directories_created(self, dirs): + import params - ignored_dfs_dirs = HdfsResourceProvider.get_ignored_resources_list(params.hdfs_resource_ignore_file) + ignored_dfs_dirs = HdfsResourceProvider.get_ignored_resources_list(params.hdfs_resource_ignore_file) - if params.security_enabled: - Execute(format("{kinit_path_local} -kt {livy_kerberos_keytab} {livy2_principal}"), - user=params.livy2_user - ) - Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"), - user=params.hdfs_user - ) + if params.security_enabled: + Execute(format("{kinit_path_local} -kt {livy_kerberos_keytab} {livy2_principal}"), + user=params.livy2_user + ) + Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"), + user=params.hdfs_user + ) - for dir_path in dirs: - self.wait_for_dfs_directory_created(dir_path, ignored_dfs_dirs) + for dir_path in dirs: + self.wait_for_dfs_directory_created(dir_path, ignored_dfs_dirs) - def get_pid_files(self): - import status_params - return [status_params.livy2_server_pid_file] + def get_pid_files(self): + import status_params + return [status_params.livy2_server_pid_file] - @retry(times=8, sleep_time=20, backoff_factor=1, err_class=Fail) - def wait_for_dfs_directory_created(self, dir_path, ignored_dfs_dirs): - import params + @retry(times=8, sleep_time=20, backoff_factor=1, err_class=Fail) + def wait_for_dfs_directory_created(self, dir_path, ignored_dfs_dirs): + import params + if not is_empty(dir_path): + dir_path = HdfsResourceProvider.parse_path(dir_path) - if not is_empty(dir_path): - dir_path = HdfsResourceProvider.parse_path(dir_path) + if dir_path in ignored_dfs_dirs: + Logger.info("Skipping DFS directory '" + dir_path + "' as it's marked to be ignored.") + return - if dir_path in ignored_dfs_dirs: - Logger.info("Skipping DFS directory '" + dir_path + "' as it's marked to be ignored.") - return + Logger.info("Verifying if DFS directory '" + dir_path + "' exists.") - Logger.info("Verifying if DFS directory '" + dir_path + "' exists.") + dir_exists = None - dir_exists = None + if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs): + # check with webhdfs is much faster than executing hdfs dfs -test + util = WebHDFSUtil(params.hdfs_site, params.hdfs_user, params.security_enabled) + list_status = util.run_command(dir_path, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False) + dir_exists = ('FileStatus' in list_status) + else: + # have to do time expensive hdfs dfs -d check. + dfs_ret_code = shell.call(format("hdfs --config {hadoop_conf_dir} dfs -test -d " + dir_path), user=params.livy_user)[0] + dir_exists = not dfs_ret_code #dfs -test -d returns 0 in case the dir exists - if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs): - # check with webhdfs is much faster than executing hdfs dfs -test - util = WebHDFSUtil(params.hdfs_site, params.hdfs_user, params.security_enabled) - list_status = util.run_command(dir_path, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False) - dir_exists = ('FileStatus' in list_status) - else: - # have to do time expensive hdfs dfs -d check. - dfs_ret_code = shell.call(format("hdfs --config {hadoop_conf_dir} dfs -test -d " + dir_path), user=params.livy_user)[0] - dir_exists = not dfs_ret_code #dfs -test -d returns 0 in case the dir exists + if not dir_exists: + raise Fail("DFS directory '" + dir_path + "' does not exist !") + else: + Logger.info("DFS directory '" + dir_path + "' exists.") - if not dir_exists: - raise Fail("DFS directory '" + dir_path + "' does not exist !") - else: - Logger.info("DFS directory '" + dir_path + "' exists.") + def get_component_name(self): + return "livy2-server" - def get_component_name(self): - return "livy2-server" + def pre_upgrade_restart(self, env, upgrade_type=None): + import params - def pre_upgrade_restart(self, env, upgrade_type=None): - import params + env.set_params(params) + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + Logger.info("Executing Livy2 Server Stack Upgrade pre-restart") + conf_select.select(params.stack_name, "spark2", params.version) + stack_select.select("livy2-server", params.version) - env.set_params(params) - if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): - Logger.info("Executing Livy2 Server Stack Upgrade pre-restart") - conf_select.select(params.stack_name, "spark2", params.version) - stack_select.select("livy2-server", params.version) + def get_log_folder(self): + import params + return params.livy2_log_dir - def get_log_folder(self): - import params - return params.livy2_log_dir - - def get_user(self): - import params - return params.livy2_user + def get_user(self): + import params + return params.livy2_user if __name__ == "__main__": LivyServer().execute() diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py index e90f6744a51..0d60cf41adb 100644 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py @@ -23,24 +23,25 @@ import threading def livy2_service(name, upgrade_type=None, action=None): - import params - - if action == 'start': - livyserver_no_op_test = format( - 'ls {livy2_server_pid_file} >/dev/null 2>&1 && ps -p `cat {livy2_server_pid_file}` >/dev/null 2>&1') - Execute(format('{livy2_server_start}'), - user=params.livy2_user, - environment={'JAVA_HOME': params.java_home}, - not_if=livyserver_no_op_test) - - elif action == 'stop': - Execute(format('{livy2_server_stop}'), - user=params.livy2_user, - environment={'JAVA_HOME': params.java_home} - ) - File(params.livy2_server_pid_file, - action="delete" - ) + import params + + if action == 'start': + livyserver_no_op_test = format( + 'ls {livy2_server_pid_file} >/dev/null 2>&1 && ps -p `cat {livy2_server_pid_file}` >/dev/null 2>&1') + Execute(format('{livy2_server_start}'), + user=params.livy2_user, + environment={'JAVA_HOME': params.java_home}, + not_if=livyserver_no_op_test + ) + + elif action == 'stop': + Execute(format('{livy2_server_stop}'), + user=params.livy2_user, + environment={'JAVA_HOME': params.java_home} + ) + File(params.livy2_server_pid_file, + action="delete" + ) diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py index ab44cf8f35d..f2178f71757 100644 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py @@ -23,58 +23,58 @@ def setup_livy(env, type, upgrade_type = None, action = None): - import params + import params - Directory([params.livy2_pid_dir, params.livy2_log_dir], - owner=params.livy2_user, - group=params.user_group, - mode=0775, - create_parents = True - ) - if type == 'server' and action == 'config': - params.HdfsResource(params.livy2_hdfs_user_dir, - type="directory", - action="create_on_execute", - owner=params.livy2_user, - mode=0775 - ) - params.HdfsResource(None, action="execute") + Directory([params.livy2_pid_dir, params.livy2_log_dir], + owner=params.livy2_user, + group=params.user_group, + mode=0775, + create_parents = True + ) + if type == 'server' and action == 'config': + params.HdfsResource(params.livy2_hdfs_user_dir, + type="directory", + action="create_on_execute", + owner=params.livy2_user, + mode=0775 + ) + params.HdfsResource(None, action="execute") - # create livy-env.sh in etc/conf dir - File(os.path.join(params.livy2_conf, 'livy-env.sh'), - owner=params.livy2_user, - group=params.livy2_group, - content=InlineTemplate(params.livy2_env_sh), - mode=0644, - ) + # create livy-env.sh in etc/conf dir + File(os.path.join(params.livy2_conf, 'livy-env.sh'), + owner=params.livy2_user, + group=params.livy2_group, + content=InlineTemplate(params.livy2_env_sh), + mode=0644, + ) - # create livy.conf in etc/conf dir - PropertiesFile(format("{livy2_conf}/livy.conf"), - properties = params.config['configurations']['livy2-conf'], - key_value_delimiter = " ", - owner=params.livy2_user, - group=params.livy2_group, - ) + # create livy.conf in etc/conf dir + PropertiesFile(format("{livy2_conf}/livy.conf"), + properties = params.config['configurations']['livy2-conf'], + key_value_delimiter = " ", + owner=params.livy2_user, + group=params.livy2_group, + ) - # create log4j.properties in etc/conf dir - File(os.path.join(params.livy2_conf, 'log4j.properties'), - owner=params.livy2_user, - group=params.livy2_group, - content=params.livy2_log4j_properties, - mode=0644, - ) + # create log4j.properties in etc/conf dir + File(os.path.join(params.livy2_conf, 'log4j.properties'), + owner=params.livy2_user, + group=params.livy2_group, + content=params.livy2_log4j_properties, + mode=0644, + ) - # create spark-blacklist.properties in etc/conf dir - File(os.path.join(params.livy2_conf, 'spark-blacklist.conf'), - owner=params.livy2_user, - group=params.livy2_group, - content=params.livy2_spark_blacklist_properties, - mode=0644, - ) + # create spark-blacklist.properties in etc/conf dir + File(os.path.join(params.livy2_conf, 'spark-blacklist.conf'), + owner=params.livy2_user, + group=params.livy2_group, + content=params.livy2_spark_blacklist_properties, + mode=0644, + ) - Directory(params.livy2_logs_dir, - owner=params.livy2_user, - group=params.livy2_group, - mode=0755, - ) + Directory(params.livy2_logs_dir, + owner=params.livy2_user, + group=params.livy2_group, + mode=0755, + ) diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml index f2a2316b3e7..457869d74a2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml @@ -27,7 +27,7 @@ mode, Livy will enable debugging options, such as reporting possible routes on a 404. defaults to development - + livy.server.port @@ -35,7 +35,7 @@ What port to start the server on. Defaults to 8999. - + livy.server.session.timeout @@ -44,7 +44,7 @@ Time in milliseconds on how long Livy will wait before timing out an idle session. Default is one hour. - + livy.impersonation.enabled @@ -52,7 +52,7 @@ If livy should use proxy users when submitting a job. - + livy.impersonation.enabled @@ -60,7 +60,7 @@ If livy should use proxy users when submitting a job. - + livy.server.csrf_protection.enabled @@ -68,7 +68,7 @@ Whether to enable csrf protection for livy's rest api. - + livy.spark.master @@ -76,7 +76,7 @@ spark.master property for spark engine - + diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml index 71b1f21a9e1..6ba9141f3c9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml @@ -29,7 +29,7 @@ user false - + livy2_group @@ -40,7 +40,7 @@ user - + livy2_log_dir @@ -49,7 +49,7 @@ directory - + livy2_pid_dir @@ -57,7 +57,7 @@ directory - + spark_home @@ -65,7 +65,7 @@ directory - + @@ -89,7 +89,7 @@ content - + diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-log4j-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-log4j-properties.xml index 64e3c0ca34c..ccc1dcfb593 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-log4j-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-log4j-properties.xml @@ -36,7 +36,7 @@ content false - + diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-spark-blacklist.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-spark-blacklist.xml index 8319b21ffd7..43b312aa5bb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-spark-blacklist.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-spark-blacklist.xml @@ -47,6 +47,6 @@ content false - + \ No newline at end of file From c0bc8ea1d8ad57806fba0bcd8362c48374fd0a9d Mon Sep 17 00:00:00 2001 From: Mingjie Tang Date: Thu, 5 Jan 2017 16:16:44 -0800 Subject: [PATCH 12/12] update the hardcode of hdp stack address --- .../stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml index 6ba9141f3c9..afb1d17e852 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-env.xml @@ -61,7 +61,7 @@ spark_home - /usr/hdp/current/spark2-client + {{stack_root}}/current/spark2-client directory