Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

HUE-662. [config] Each Hadoop hdfs/mr/yarn cluster to define its own env

* Have Beeswax and Shell define the appropriate HADOOP_MAPRED_HOME for the
  child process.
* Renamed JT_HOST & RM_HOST to just "HOST", and JT_PORT & RM_PORT to just
  "PORT" for consistent reference.
* Fixed README to drop the requirement on having a HADOOP_HOME. The build
  does not depend on one because we're using maven.
  • Loading branch information...
commit c7e29b51f7ac4e1ab020583505e7a28c9c83b901 1 parent 871a084
bc Wong authored
View
2  README.testing
@@ -84,7 +84,7 @@ DESKTOP_DB_CONFIG=<db engine:db name:test db name:username:password:host:port>
Writing tests that depend on Hadoop
===================================
-Use mini_cluster.py! You should tag such tests with "requires_hadoop", as follows:
+Use pseudo_hdfs4.py! You should tag such tests with "requires_hadoop", as follows:
from nose.plugins.attrib import attr
View
11 apps/beeswax/beeswax_server.sh
@@ -19,8 +19,8 @@
set -o errexit
-if [ -z "$HADOOP_HOME" ]; then
- echo "\$HADOOP_HOME must be specified" 1>&2
+if [ -z "$HADOOP_CONF_DIR" ]; then
+ echo "\$HADOOP_CONF_DIR must be specified" 1>&2
exit 1
fi
echo \$HADOOP_HOME=$HADOOP_HOME
@@ -46,7 +46,6 @@ fi
echo \$HIVE_HOME=$HIVE_HOME
-
BEESWAX_ROOT=$(dirname $0)
BEESWAX_JAR=$BEESWAX_ROOT/java-lib/BeeswaxServer.jar
HIVE_LIB=$HIVE_HOME/lib
@@ -65,15 +64,13 @@ echo \$HADOOP_OPTS=$HADOOP_OPTS
# and to force hive-default to correspond to the Hive version we have.
# Because we are abusing HADOOP_CONF_DIR, we have to emulate its default
# behavior here as well.
-if [ -z "$HADOOP_CONF_DIR" ]; then
- HADOOP_CONF_DIR="$HADOOP_HOME/conf"
-fi
if [ -f $HADOOP_CONF_DIR/hadoop-env.sh ]; then
. $HADOOP_CONF_DIR/hadoop-env.sh
fi
-export HADOOP_CONF_DIR=$HIVE_CONF_DIR:$BEESWAX_ROOT/../../desktop/conf:$HADOOP_CONF_DIR
+export HADOOP_CONF_DIR=$HIVE_CONF_DIR:$HADOOP_CONF_DIR
echo \$HADOOP_CONF_DIR=$HADOOP_CONF_DIR
+echo \$HADOOP_MAPRED_HOME=$HADOOP_MAPRED_HOME
# Note: I've had trouble running this with just "java -jar" with the classpath
# determined with a seemingly appropriate find command.
View
34 apps/beeswax/src/beeswax/management/commands/beeswax_server.py
@@ -22,26 +22,36 @@
import beeswax.conf
import beeswax.hive_site
import desktop.conf
-import hadoop.conf
-import os
+import hadoop.cluster
+
import logging
+import os
+import sys
LOG = logging.getLogger(__name__)
class Command(NoArgsCommand):
""" Starts beeswax daemon. """
+
def handle_noargs(self, **options):
+ cluster_conf = hadoop.cluster.get_cluster_conf_for_job_submission()
+ if cluster_conf is None:
+ LOG.error("Configuration does not contain any MR/Yarn clusters with "
+ "`submit_to' enabled. Cannot start BeeswaxServer.")
+ sys.exit(1)
+
env = os.environ.copy()
- env['HADOOP_HOME'] = hadoop.conf.HADOOP_HOME.get()
- env['HADOOP_BIN'] = hadoop.conf.HADOOP_BIN.get()
- if hadoop.conf.HADOOP_CONF_DIR.get():
- env['HADOOP_CONF_DIR'] = hadoop.conf.HADOOP_CONF_DIR.get()
- if beeswax.conf.BEESWAX_HIVE_HOME_DIR.get():
- env['HIVE_HOME'] = beeswax.conf.BEESWAX_HIVE_HOME_DIR.get()
- if beeswax.conf.BEESWAX_HIVE_CONF_DIR.get():
- env['HIVE_CONF_DIR'] = beeswax.conf.BEESWAX_HIVE_CONF_DIR.get()
- if beeswax.conf.BEESWAX_SERVER_HEAPSIZE.get():
- env['HADOOP_HEAPSIZE'] = beeswax.conf.BEESWAX_SERVER_HEAPSIZE.get()
+ def set_if_present(name, val):
+ if val:
+ env[name] = val
+
+ env['HADOOP_BIN'] = cluster_conf.HADOOP_BIN.get()
+ set_if_present('HADOOP_MAPRED_HOME', cluster_conf.HADOOP_MAPRED_HOME.get())
+ set_if_present('HADOOP_CONF_DIR', cluster_conf.HADOOP_CONF_DIR.get())
+ set_if_present('HADOOP_HEAPSIZE', beeswax.conf.BEESWAX_SERVER_HEAPSIZE.get())
+ set_if_present('HIVE_HOME', beeswax.conf.BEESWAX_HIVE_HOME_DIR.get())
+ set_if_present('HIVE_CONF_DIR', beeswax.conf.BEESWAX_HIVE_CONF_DIR.get())
+
bin = beeswax.conf.BEESWAX_SERVER_BIN.get()
# Host that desktop is running on
View
2  apps/jobsub/src/jobsub/submit.py
@@ -69,7 +69,7 @@ def run(self):
raise Exception("Job design already submitted (Oozie job id %s)" % (self.job_id,))
fs_defaultfs = self._fs.fs_defaultfs
- jobtracker = hadoop.cluster.get_cluster_for_job_submission()
+ jobtracker = hadoop.cluster.get_cluster_addr_for_job_submission()
try:
wf_dir = self._get_and_create_deployment_dir()
View
1  apps/shell/src/shell/constants.py
@@ -49,6 +49,7 @@
SHELL_NOT_ALLOWED = "shellNotAllowed"
HOME = "HOME"
HADOOP_HOME = "HADOOP_HOME"
+HADOOP_MAPRED_HOME = "HADOOP_MAPRED_HOME"
HADOOP_TOKEN_FILE_LOCATION = 'HADOOP_TOKEN_FILE_LOCATION'
EXISTS = "exists"
View
13 apps/shell/src/shell/shellmanager.py
@@ -21,7 +21,6 @@
import cStringIO
-import desktop.lib.i18n
import errno
import eventlet
import hadoop.conf
@@ -39,7 +38,8 @@
from eventlet.green import os
from eventlet.green import select
from eventlet.green import time
-from hadoop.cluster import all_mrclusters, get_all_hdfs
+from hadoop.cluster import all_mrclusters, get_all_hdfs, \
+ get_cluster_conf_for_job_submission
LOG = logging.getLogger(__name__)
SHELL_OUTPUT_LOGGER = logging.getLogger("shell_output")
@@ -388,6 +388,7 @@ def __init__(self):
self._parse_configs()
eventlet.spawn_after(1, self._handle_periodic)
+
@classmethod
def global_instance(cls):
if not hasattr(cls, "_global_instance"):
@@ -395,9 +396,15 @@ def global_instance(cls):
return cls._global_instance
def _parse_configs(self):
+ mr_cluster = get_cluster_conf_for_job_submission()
+ if mr_cluster is None:
+ mapred_home = None
+ else:
+ mapred_home = mr_cluster.HADOOP_MAPRED_HOME.get()
+
shell_types = [] # List of available shell types. For each shell type, we have a nice name (e.g. "Python Shell") and a short name (e.g. "python")
for item in shell.conf.SHELL_TYPES.keys():
- env_for_shell = { constants.HADOOP_HOME: hadoop.conf.HADOOP_HOME.get() }
+ env_for_shell = { constants.HADOOP_MAPRED_HOME: mapred_home }
command = shell.conf.SHELL_TYPES[item].command.get().strip().split()
nice_name = shell.conf.SHELL_TYPES[item].nice_name.get().strip()
executable_exists = utils.executable_exists(command)
View
52 desktop/conf.dist/hue.ini
@@ -165,15 +165,6 @@
[hadoop]
- # If you installed Hadoop in a different location, you need to set
- # hadoop_home, in which bin/hadoop, the Hadoop wrapper script, is found.
- #
- # NOTE: Hue depends on Cloudera's Distribution of Hadoop version 4 (CDH4)
- # or later.
- hadoop_home=/usr/lib/hadoop
- hadoop_bin=/usr/bin/hadoop
- # hadoop_conf_dir=/etc/hadoop/conf
-
# Configuration for HDFS NameNode
# ------------------------------------------------------------------------
[[hdfs_clusters]]
@@ -185,11 +176,23 @@
# Change this if your HDFS cluster is Kerberos-secured
## security_enabled=false
- # Use WebHdfs/HttpFs as the communication mechanism. To fallback to
- # using the Thrift plugin (used in Hue 1.x), this must be uncommented
- # and explicitly set to the empty value.
+ # Use WebHdfs/HttpFs as the communication mechanism.
+ # This should be the web service root URL, such as
+ # http://namenode:50070/webhdfs/v1
## webhdfs_url=
+ # Settings about this HDFS cluster. If you install HDFS in a
+ # different location, you need to set the following.
+
+ # Defaults to $HADOOP_HDFS_HOME or /usr/lib/hadoop-hdfs
+ ## hadoop_hdfs_home=/usr/lib/hadoop-hdfs
+
+ # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+ ## hadoop_bin=/usr/bin/hadoop
+
+ # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+ ## hadoop_conf_dir=/etc/hadoop/conf
+
# Configuration for MapReduce 0.20 JobTracker (MR1)
# ------------------------------------------------------------------------
[[mapred_clusters]]
@@ -202,11 +205,22 @@
# Thrift plug-in port for the JobTracker
## thrift_port=9290
# Whether to submit jobs to this cluster
- ## submit_to=False
+ ## submit_to=True
# Change this if your MapReduce cluster is Kerberos-secured
## security_enabled=false
+ # Settings about this MR1 cluster. If you install MR1 in a
+ # different location, you need to set the following.
+
+ # Defaults to $HADOOP_MR1_HOME or /usr/lib/hadoop-0.20-mapreduce
+ ## hadoop_mapred_home=/usr/lib/hadoop-0.20-mapreduce
+
+ # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+ ## hadoop_bin=/usr/bin/hadoop
+
+ # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+ ## hadoop_conf_dir=/etc/hadoop/conf
# Configuration for Yarn (MR2)
# ------------------------------------------------------------------------
@@ -220,6 +234,18 @@
# Whether to submit jobs to this cluster
## submit_to=False
+ # Settings about this MR2 cluster. If you install MR2 in a
+ # different location, you need to set the following.
+
+ # Defaults to $HADOOP_MR2_HOME or /usr/lib/hadoop-mapreduce
+ ## hadoop_mapred_home=/usr/lib/hadoop-mapreduce
+
+ # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+ ## hadoop_bin=/usr/bin/hadoop
+
+ # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+ ## hadoop_conf_dir=/etc/hadoop/conf
+
###########################################################################
# Settings to configure Beeswax
View
47 desktop/conf/pseudo-distributed.ini.tmpl
@@ -171,17 +171,6 @@
[hadoop]
- # If you installed Hadoop in a different location, you need to set
- # hadoop_home, in which bin/hadoop, the Hadoop wrapper script, is found.
- #
- # NOTE: Hue depends on Cloudera's Distribution of Hadoop version 4 (CDH4)
- # or later.
- hadoop_home=$HADOOP_HOME
- hadoop_bin=$HADOOP_HOME/bin/hadoop
- hadoop_conf_dir=$HADOOP_HOME/conf
-
- ## hadoop_mr1_home=$HADOOP_MR1_HOME
-
# Configuration for HDFS NameNode
# ------------------------------------------------------------------------
[[hdfs_clusters]]
@@ -195,6 +184,18 @@
# and explicitly set to the empty value.
## webhdfs_url=
+ # Settings about this HDFS cluster. If you install HDFS in a
+ # different location, you need to set the following.
+
+ # Defaults to $HADOOP_HDFS_HOME or /usr/lib/hadoop-hdfs
+ ## hadoop_hdfs_home=/usr/lib/hadoop-hdfs
+
+ # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+ ## hadoop_bin=/usr/bin/hadoop
+
+ # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+ ## hadoop_conf_dir=/etc/hadoop/conf
+
# Configuration for MapReduce JobTracker
# ------------------------------------------------------------------------
[[mapred_clusters]]
@@ -209,6 +210,18 @@
# Whether to submit jobs to this cluster
## submit_to=False
+ # Settings about this MR1 cluster. If you install MR1 in a
+ # different location, you need to set the following.
+
+ # Defaults to $HADOOP_MR1_HOME or /usr/lib/hadoop-0.20-mapreduce
+ ## hadoop_mapred_home=/usr/lib/hadoop-0.20-mapreduce
+
+ # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+ ## hadoop_bin=/usr/bin/hadoop
+
+ # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+ ## hadoop_conf_dir=/etc/hadoop/conf
+
# Configuration for Yarn
# ------------------------------------------------------------------------
[[yarn_clusters]]
@@ -221,6 +234,18 @@
# Whether to submit jobs to this cluster
## submit_to=False
+ # Settings about this MR2 cluster. If you install MR2 in a
+ # different location, you need to set the following.
+
+ # Defaults to $HADOOP_MR2_HOME or /usr/lib/hadoop-mapreduce
+ ## hadoop_mapred_home=/usr/lib/hadoop-mapreduce
+
+ # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+ ## hadoop_bin=/usr/bin/hadoop
+
+ # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+ ## hadoop_conf_dir=/etc/hadoop/conf
+
###########################################################################
# Settings to configure Beeswax
View
19 desktop/libs/hadoop/src/hadoop/cluster.py
@@ -91,21 +91,32 @@ def all_mrclusters():
MR_CACHE[identifier] = _make_mrcluster(identifier)
return MR_CACHE
-def get_cluster_for_job_submission():
+def get_cluster_conf_for_job_submission():
"""
Check the `submit_to' for each MR/Yarn cluster, and return the
- host:port of first one that enables submission.
+ config section of first one that enables submission.
"""
for name in conf.YARN_CLUSTERS.keys():
yarn = conf.YARN_CLUSTERS[name]
if yarn.SUBMIT_TO.get():
- return "%s:%s" % (yarn.RM_HOST.get(), yarn.RM_PORT.get())
+ return yarn
for name in conf.MR_CLUSTERS.keys():
mr = conf.MR_CLUSTERS[name]
if mr.SUBMIT_TO.get():
- return "%s:%s" % (mr.JT_HOST.get(), mr.JT_PORT.get())
+ return mr
return None
+def get_cluster_addr_for_job_submission():
+ """
+ Check the `submit_to' for each MR/Yarn cluster, and return the
+ host:port of first one that enables submission.
+ """
+ conf = get_cluster_conf_for_job_submission()
+ if conf is None:
+ return None
+ return "%s:%s" % (conf.HOST.get(), conf.PORT.get())
+
+
def clear_caches():
"""
Clears cluster's internal caches. Returns
View
184 desktop/libs/hadoop/src/hadoop/conf.py
@@ -22,46 +22,9 @@
DEFAULT_NN_HTTP_PORT = 50070
-HADOOP_HOME = Config(
- key="hadoop_home",
- default=os.environ.get("HADOOP_HOME", "/usr/lib/hadoop"),
- help=("Path to directory holding hadoop libs - HADOOP_HOME in " +
- "hadoop parlance; defaults to environment variable, when" +
- "set.")
-)
-
-HADOOP_MR1_HOME = Config(
- key="hadoop_mr1_home",
- default=os.environ.get("HADOOP_MR1_HOME", "/usr/lib/hadoop-0.20-mapreduce"),
- help=("Path to directory holding hadoop libs - HADOOP_HOME in " +
- "hadoop parlance; defaults to environment variable, when" +
- "set.")
-)
-
-def hadoop_bin_from_hadoop_home():
- """Returns $HADOOP_HOME/bin/hadoop"""
- return os.path.join(HADOOP_HOME.get(), "bin/hadoop")
-
-HADOOP_BIN = Config("hadoop_bin",
- help="Path to your Hadoop binary",
- dynamic_default=hadoop_bin_from_hadoop_home,
- type=str)
-
-# TODO(philip): This will need more love for dealing with multiple clusters.
-HADOOP_CONF_DIR = Config(
- key="hadoop_conf_dir",
- default="/etc/hadoop/conf",
- help="Directory to pass to hadoop_bin (from Hadoop configuration) as the --config flag.",
-)
-
-def find_file_recursive(desired_glob, root=None):
- if root is None:
- root_f = lambda: HADOOP_HOME.get()
- else:
- root_f = lambda: not callable(root) and root or root()
-
+def find_file_recursive(desired_glob, root):
def f():
- for dirpath, dirnames, filenames in os.walk(root_f()):
+ for dirpath, dirnames, filenames in os.walk(root):
matches = fnmatch.filter(filenames, desired_glob)
if matches:
if len(matches) != 1:
@@ -72,33 +35,9 @@ def f():
logging.error("Trouble finding jars matching %s" % (desired_glob,))
return None
- if root is None:
- root_str = "$HADOOP_HOME"
- else:
- root_str = root
- f.__doc__ = "Finds %s/%s" % (root_str, desired_glob)
+ f.__doc__ = "Finds %s/%s" % (root, desired_glob)
return f
-HADOOP_EXAMPLES_JAR = Config(
- key="hadoop_examples_jar",
- dynamic_default=find_file_recursive("hadoop-*examples*.jar", lambda: HADOOP_MR1_HOME.get()),
- help="Path to the hadoop-examples.jar (used for tests and jobdesigner setup)",
- type=str,
- private=True)
-
-HADOOP_STREAMING_JAR = Config(
- key="hadoop_streaming_jar",
- dynamic_default=find_file_recursive("hadoop-*streaming*.jar", lambda: HADOOP_MR1_HOME.get()),
- help="Path to the hadoop-streaming.jar (used by jobdesigner)",
- type=str,
- private=True)
-
-HADOOP_TEST_JAR = Config("hadoop_test_jar",
- help="[Used by testing code.] Path to hadoop-test.jar",
- dynamic_default=find_file_recursive("hadoop-*test*.jar", lambda: HADOOP_MR1_HOME.get()),
- type=str,
- private=True)
-
HADOOP_PLUGIN_CLASSPATH = Config("hadoop_plugin_classpath",
help="[Used only in testing code.] Path to the Hadoop plugin jar.",
type=str,
@@ -149,6 +88,31 @@ def f():
default=False, type=coerce_bool),
TEMP_DIR=Config("temp_dir", help="HDFS directory for temporary files",
default='/tmp', type=str),
+
+ HADOOP_HDFS_HOME = Config(
+ key="hadoop_hdfs_home",
+ default=os.environ.get("HADOOP_HDFS_HOME", "/usr/lib/hadoop-hdfs"),
+ help=("Path to Hadoop HDFS home - HADOOP_HOME or HADOOP_HDFS_HOME in " +
+ "hadoop parlance. For tarball installations, it is the root of " +
+ "the untarred directory. For packages, " +
+ "it is /usr/lib/hadoop-hdfs." +
+ "Defaults to the environment varible HADOOP_BIN when set, " +
+ "or '/usr/bin/hadoop'."),
+ ),
+ HADOOP_BIN = Config(
+ key="hadoop_bin",
+ default=os.environ.get("HADOOP_BIN", "/usr/bin/hadoop"),
+ help=("Path to your Hadoop launcher script. E.g. /usr/bin/hadoop. " +
+ "Defaults to the environment varible HADOOP_BIN when set, " +
+ "or '/usr/bin/hadoop'.")
+ ),
+ HADOOP_CONF_DIR = Config(
+ key="hadoop_conf_dir",
+ default=os.environ.get("HADOOP_CONF_DIR", "/etc/hadoop/conf"),
+ help=("Directory to pass to hadoop_bin (from Hadoop configuration) " +
+ "as the --config flag. Defaults to the environment variable " +
+ "HADOOP_CONF_DIR when set, or '/etc/hadoop/conf'.")
+ ),
)
)
)
@@ -159,11 +123,11 @@ def f():
each=ConfigSection(
help="Information about a single MapReduce cluster",
members=dict(
- JT_HOST=Config("jobtracker_host", help="Host/IP for JobTracker"),
- JT_PORT=Config("jobtracker_port",
- default=8021,
- help="Service port for the JobTracker",
- type=int),
+ HOST=Config("jobtracker_host", help="Host/IP for JobTracker"),
+ PORT=Config("jobtracker_port",
+ default=8021,
+ help="Service port for the JobTracker",
+ type=int),
JT_THRIFT_PORT=Config("thrift_port", help="Thrift port for JobTracker", default=9290,
type=int),
JT_KERBEROS_PRINCIPAL=Config("jt_kerberos_principal", help="Kerberos principal for JobTracker",
@@ -171,7 +135,29 @@ def f():
SECURITY_ENABLED=Config("security_enabled", help="Is running with Kerberos authentication",
default=False, type=coerce_bool),
SUBMIT_TO=Config('submit_to', help="Whether Hue should use this cluster to run jobs",
- default=False, type=coerce_bool),
+ default=True, type=coerce_bool),
+
+ HADOOP_MAPRED_HOME = Config(
+ key="hadoop_mapred_home",
+ default=os.environ.get("HADOOP_MR1_HOME", "/usr/lib/hadoop-0.20-mapreduce"),
+ help=("Path to directory holding Hadoop MR1 libs. " +
+ "E.g. /usr/lib/hadoop. Defaults to the environment variable " +
+ "HADOOP_MR1_HOME when set, or '/usr/lib/hadoop'.")
+ ),
+ HADOOP_BIN = Config(
+ key="hadoop_bin",
+ default=os.environ.get("HADOOP_MR1_BIN", "/usr/bin/hadoop"),
+ help=("Path to your Hadoop launcher script. E.g. /usr/bin/hadoop. " +
+ "Defaults to the environment varible HADOOP_MR1_BIN when set, " +
+ "or '/usr/bin/hadoop'.")
+ ),
+ HADOOP_CONF_DIR = Config(
+ key="hadoop_conf_dir",
+ default=os.environ.get("HADOOP_CONF_DIR", "/etc/hadoop/conf"),
+ help=("Directory to pass to hadoop_bin (from Hadoop configuration) " +
+ "as the --config flag. Defaults to the environment variable " +
+ "HADOOP_CONF_DIR when set, or '/etc/hadoop/conf'.")
+ ),
)
)
)
@@ -183,15 +169,37 @@ def f():
each=ConfigSection(
help="Information about a single Yarn cluster",
members=dict(
- RM_HOST=Config("resourcemanager_host",
- default='localhost',
- help="Host/IP for the ResourceManager"),
- RM_PORT=Config("resourcemanager_port",
- default=8032,
- type=int,
- help="Service port for the ResourceManager"),
+ HOST=Config("resourcemanager_host",
+ default='localhost',
+ help="Host/IP for the ResourceManager"),
+ PORT=Config("resourcemanager_port",
+ default=8032,
+ type=int,
+ help="Service port for the ResourceManager"),
SUBMIT_TO=Config('submit_to', help="Whether Hue should use this cluster to run jobs",
default=False, type=coerce_bool),
+
+ HADOOP_MAPRED_HOME = Config(
+ key="hadoop_mapred_home",
+ default=os.environ.get("HADOOP_MR2_HOME", "/usr/lib/hadoop-mapreduce"),
+ help=("Path to directory holding Hadoop MR2 libs. " +
+ "E.g. /usr/lib/hadoop. Defaults to the environment " +
+ "variable HADOOP_MR2_HOME when set, or '/usr/lib/hadoop'.")
+ ),
+ HADOOP_BIN = Config(
+ key="hadoop_bin",
+ default=os.environ.get("HADOOP_MR2_BIN", "/usr/bin/hadoop"),
+ help=("Path to your Hadoop launcher script. E.g. /usr/bin/hadoop. " +
+ "Defaults to the environment varible HADOOP_MR2_BIN when set, " +
+ "or '/usr/bin/hadoop'.")
+ ),
+ HADOOP_CONF_DIR = Config(
+ key="hadoop_conf_dir",
+ default=os.environ.get("HADOOP_CONF_DIR", "/etc/hadoop/conf"),
+ help=("Directory to pass to hadoop_bin (from Hadoop configuration) " +
+ "as the --config flag. Defaults to the environment variable " +
+ "HADOOP_CONF_DIR when set, or '/etc/hadoop/conf'.")
+ ),
)
)
)
@@ -206,22 +214,15 @@ def config_validator():
from hadoop.fs import webhdfs
from hadoop import job_tracker
res = [ ]
-
- # HADOOP_HOME
- res.extend(validate_path(HADOOP_HOME, is_dir=True))
- # HADOOP_BIN
- res.extend(validate_path(HADOOP_BIN, is_dir=False))
-
- # JARs: even though these are private, we need them to run jobsub
- res.extend(validate_path(HADOOP_EXAMPLES_JAR, is_dir=False))
- res.extend(validate_path(HADOOP_STREAMING_JAR, is_dir=False))
-
submit_to = [ ]
# HDFS_CLUSTERS
has_default = False
for name in HDFS_CLUSTERS.keys():
cluster = HDFS_CLUSTERS[name]
+ res.extend(validate_path(cluster.HADOOP_HDFS_HOME, is_dir=True))
+ res.extend(validate_path(cluster.HADOOP_CONF_DIR, is_dir=True))
+ res.extend(validate_path(cluster.HADOOP_BIN, is_dir=False))
res.extend(webhdfs.test_fs_configuration(cluster))
if name == 'default':
has_default = True
@@ -231,16 +232,23 @@ def config_validator():
# MR_CLUSTERS
for name in MR_CLUSTERS.keys():
cluster = MR_CLUSTERS[name]
+ res.extend(validate_path(cluster.HADOOP_MAPRED_HOME, is_dir=True))
+ res.extend(validate_path(cluster.HADOOP_CONF_DIR, is_dir=True))
+ res.extend(validate_path(cluster.HADOOP_BIN, is_dir=False))
res.extend(job_tracker.test_jt_configuration(cluster))
if cluster.SUBMIT_TO.get():
submit_to.append('mapred_clusters.' + name)
- # Only one cluster should have submit_to
+ # YARN_CLUSTERS
for name in YARN_CLUSTERS.keys():
cluster = YARN_CLUSTERS[name]
+ res.extend(validate_path(cluster.HADOOP_MAPRED_HOME, is_dir=True))
+ res.extend(validate_path(cluster.HADOOP_CONF_DIR, is_dir=True))
+ res.extend(validate_path(cluster.HADOOP_BIN, is_dir=False))
if cluster.SUBMIT_TO.get():
submit_to.append('yarn_clusters.' + name)
+ # Only one cluster should have submit_to
if len(submit_to) > 1:
res.append(("hadoop", "Only one cluster may enable 'submit_to'. "
"But it is enabled in the following clusters: " +
View
4 desktop/libs/hadoop/src/hadoop/job_tracker.py
@@ -50,7 +50,7 @@ def test_jt_configuration(cluster):
jt.runtime_info()
except TTransport.TTransportException:
msg = 'Failed to contact JobTracker plugin at %s:%s.' % \
- (cluster.JT_HOST.get(), cluster.JT_THRIFT_PORT.get())
+ (cluster.HOST.get(), cluster.JT_THRIFT_PORT.get())
return [ (cluster, msg) ]
return []
@@ -84,7 +84,7 @@ def __init__(self, host, thrift_port,
@classmethod
def from_conf(cls, conf):
return cls(
- conf.JT_HOST.get(),
+ conf.HOST.get(),
conf.JT_THRIFT_PORT.get(),
security_enabled=conf.SECURITY_ENABLED.get(),
kerberos_principal=conf.JT_KERBEROS_PRINCIPAL.get())
View
13 desktop/libs/hadoop/src/hadoop/mini_cluster.py
@@ -14,7 +14,16 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
+
+
+#######################################################
+## WARNING!!! ##
+## This file is stale. Hadoop 0.23 and CDH4 ##
+## do not support minicluster. This is replaced ##
+## by webhdfs.py, to set up a running cluster. ##
+#######################################################
+
+
# A Python-side driver for MiniHadoopClusterManager
#
# See README.testing for hints on how to use this,
@@ -413,7 +422,7 @@ def shared_cluster(conf=False):
closers.extend([
hadoop.conf.HDFS_CLUSTERS["default"].NN_HOST.set_for_testing("localhost"),
hadoop.conf.HDFS_CLUSTERS["default"].NN_HDFS_PORT.set_for_testing(cluster.namenode_port),
- hadoop.conf.MR_CLUSTERS["default"].JT_HOST.set_for_testing("localhost"),
+ hadoop.conf.MR_CLUSTERS["default"].HOST.set_for_testing("localhost"),
hadoop.conf.MR_CLUSTERS["default"].JT_THRIFT_PORT.set_for_testing(cluster.jt.thrift_port),
])
# Clear the caches
View
28 desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py
@@ -199,7 +199,8 @@ def start(self):
# More stuff to setup in the environment
env = dict(
- HADOOP_HOME = hadoop.conf.HADOOP_HOME.get(),
+ HADOOP_HOME = hadoop.conf.HDFS_CLUSTERS['default'].HADOOP_HDFS_HOME.get(),
+ HADOOP_BIN = hadoop.conf.HDFS_CLUSTERS['default'].HADOOP_BIN.get(),
HADOOP_CONF_DIR = conf_dir,
HADOOP_HEAPSIZE = "128",
HADOOP_LOG_DIR = self._log_dir,
@@ -237,21 +238,9 @@ def _start_mr1(self, env):
# We need a different env because it's a different hadoop
env = env.copy()
- env['HADOOP_HOME'] = hadoop.conf.HADOOP_MR1_HOME.get()
+ env['HADOOP_HOME'] = hadoop.conf.MR_CLUSTERS['default'].HADOOP_MAPRED_HOME.get()
+ env['HADOOP_BIN'] = hadoop.conf.MR_CLUSTERS['default'].HADOOP_BIN.get()
env["HADOOP_CLASSPATH"] = ':'.join([
- # -- BEGIN JAVA TRIVIA --
- # Add the -test- jar to the classpath to work around a subtle issue
- # involving Java classloaders. In brief, hadoop's RunJar class creates
- # a child classloader with the test jar on it, but the core classes
- # are loaded by the system classloader. This is fine except that
- # some classes in the test jar extend package-protected classes in the
- # core jar. Even though the classes are in the same package name, they
- # are thus loaded by different classloaders and therefore an IllegalAccessError
- # prevents the MiniMRCluster from starting. Adding the test jar to the system
- # classpath prevents this error since then both the MiniMRCluster and the
- # core classes are loaded by the system classloader.
- hadoop.conf.HADOOP_TEST_JAR.get(),
- # -- END JAVA TRIVIA --
hadoop.conf.HADOOP_PLUGIN_CLASSPATH.get(),
# Due to CDH-4537, we need to add test dependencies to run minicluster
os.path.join(os.path.dirname(__file__), 'test_jars', '*'),
@@ -276,7 +265,7 @@ def _start_mr1(self, env):
def _format(self, conf_dir, env):
"""Format HDFS"""
- args = (hadoop.conf.HADOOP_BIN.get(),
+ args = (self._get_hadoop_bin(env),
'--config', conf_dir,
'namenode', '-format')
LOG.info('Formatting HDFS: %s' % (args,))
@@ -377,7 +366,10 @@ def _start_daemon(self, proc_name, conf_dir, env):
env=env)
def _get_hadoop_bin(self, env):
- return os.path.join(env['HADOOP_HOME'], 'bin', 'hadoop')
+ try:
+ return env['HADOOP_BIN']
+ except KeyError:
+ return os.path.join(env['HADOOP_HOME'], 'bin', 'hadoop')
def _write_hdfs_site(self):
self._dfs_http_port = find_unused_port()
@@ -466,7 +458,7 @@ def shared_cluster():
closers = [
hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name),
hadoop.conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing(webhdfs_url),
- hadoop.conf.MR_CLUSTERS['default'].JT_HOST.set_for_testing('localhost'),
+ hadoop.conf.MR_CLUSTERS['default'].HOST.set_for_testing('localhost'),
hadoop.conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(cluster.jt_thrift_port),
]
View
5 desktop/libs/hadoop/src/hadoop/tests.py
@@ -112,7 +112,6 @@ def test_tricky_confparse():
def test_config_validator_basic():
reset = (
- conf.HADOOP_STREAMING_JAR.set_for_testing('/tmp'),
conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing('http://not.the.re:50070/'),
conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(70000),
)
@@ -120,8 +119,6 @@ def test_config_validator_basic():
try:
cli = make_logged_in_client()
resp = cli.get('/debug/check_config')
- assert_true('hadoop.hadoop_streaming_jar' in resp.content)
- assert_true('Not a file' in resp.content)
assert_true('hadoop.hdfs_clusters.default.webhdfs_url' in resp.content)
assert_true('hadoop.mapred_clusters.default.thrift_port' in resp.content)
assert_true('Port should be' in resp.content)
@@ -141,7 +138,7 @@ def test_config_validator_more():
cli = make_logged_in_client()
reset = (
- conf.MR_CLUSTERS["default"].JT_HOST.set_for_testing("localhost"),
+ conf.MR_CLUSTERS["default"].HOST.set_for_testing("localhost"),
conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(23),
)
old = cluster.clear_caches()
View
20 dist/README
@@ -5,26 +5,18 @@ If you're impatient, these are the key steps. Please check the full manual
for more details.
## Install
-$ HADOOP_HOME=/usr/lib/hadoop-0.20 PREFIX=/usr/local make install
+$ PREFIX=/usr/share make install
## Install plug-ins
-$ cd /usr/lib/hadoop/lib
-$ ln -s /usr/local/hue/desktop/libs/hadoop/java-lib/hue*jar
+$ cd /usr/lib/hadoop-0.20-mapreduce/lib
+$ ln -s /usr/share/hue/desktop/libs/hadoop/java-lib/hue*jar
## Configure Hadoop
Edit hdfs-site.xml:
<property>
- <name>dfs.namenode.plugins</name>
- <value>org.apache.hadoop.thriftfs.NamenodePlugin</value>
- <description>Comma-separated list of namenode plug-ins to be activated.
- </description>
-</property>
-<property>
- <name>dfs.datanode.plugins</name>
- <value>org.apache.hadoop.thriftfs.DatanodePlugin</value>
- <description>Comma-separated list of datanode plug-ins to be activated.
- </description>
+ <name>dfs.webhdfs.enable</name>
+ <value>true</value>
</property>
Edit mapred-site.xml:
@@ -37,4 +29,4 @@ Edit mapred-site.xml:
</property>
## Run!
-$ /usr/local/hue/build/env/bin/supervisor
+$ /usr/share/hue/build/env/bin/supervisor
View
60 docs/manual.txt
@@ -412,19 +412,6 @@ Hue Configuration for Hadoop
These configuration variables are under the `[hadoop]` section in
the `/etc/hue/hue.ini` configuration file.
-hadoop_home::
- This becomes the value of `$HADOOP_HOME` for any Shell processes
- and the Beeswax Server. If you use MR1, set this to the MR1 home.
-
-hadoop_bin::
- Use this as the hadoop binary. If you use MR1, set this to
- `<hadoop_home>/bin/hadoop` instead of the default `/usr/bin/hadoop`.
-
-hadoop_conf_dir::
- This is the configuration directory for any processes to
- configure their Hadoop client. If you use MR1, set this to the
- directory containing your MR1 configuration.
-
HDFS Cluster
^^^^^^^^^^^^
@@ -433,13 +420,26 @@ Hue only support one HDFS cluster currently. That cluster should be defined
under the `[[[default]]]` sub-section.
fs_defaultfs::
- This is the equivalence of ``fs.defaultFS`` (aka ``fs.default.name``) in
+ This is the equivalence of `fs.defaultFS` (aka `fs.default.name`) in
Hadoop configuration.
webhdfs_url::
You can also set this to be the HttpFS url. The default value is the HTTP
port on the NameNode.
+hadoop_hdfs_home::
+ This is the home of your Hadoop HDFS installation. It is the
+ root of the Hadoop untarred directory, or usually
+ `/usr/lib/hadoop`.
+
+hadoop_bin::
+ Use this as the HDFS Hadoop launcher script, which is usually
+ `/usr/bin/hadoop`.
+
+hadoop_conf_dir::
+ This is the configuration directory of the HDFS, typically
+ `/etc/hadoop/conf`.
+
MapReduce (MR1) Cluster
^^^^^^^^^^^^^^^^^^^^^^^
@@ -457,6 +457,22 @@ submit_to::
If your Oozie is configured with to talk to a 0.20 MapReduce service, then
set this to `true`. Hue will be submitting jobs to this MapReduce cluster.
+hadoop_mapred_home::
+ This is the home of your Hadoop MapReduce installation. It is the
+ root of the Hadoop MR1 untarred directory, or the root of the
+ Hadoop 0.20 untarred directory, or `/usr/lib/hadoop-0.20-mapreduce` for
+ CDH packages. If `submit_to` is true for this cluster, this
+ config value becomes the `$HADOOP_MAPRED_HOME` for
+ BeeswaxServer and child shell processes.
+
+hadoop_bin::
+ Use this as the MR1 Hadoop launcher script, which is usually
+ `/usr/bin/hadoop`.
+
+hadoop_conf_dir::
+ This is the configuration directory of the MR1 service,
+ typically `/etc/hadoop/conf`.
+
Yarn (MR2) Cluster
^^^^^^^^^^^^^^^^^^
@@ -475,6 +491,22 @@ submit_to::
set this to `true`. Hue will be submitting jobs to this Yarn cluster.
But note that JobBrowser will not be able to show MR2 jobs.
+hadoop_mapred_home::
+ This is the home of your Hadoop MapReduce installation. It is the
+ root of the Hadoop 0.23 untarred directory, or
+ `/usr/lib/hadoop-mapreduce` for CDH packages. If `submit_to` is
+ true for this cluster, this config value becomes the
+ `$HADOOP_MAPRED_HOME` for BeeswaxServer and child shell
+ processes.
+
+hadoop_bin::
+ Use this as the Yarn/MR2 Hadoop launcher script, which is usually
+ `/usr/bin/hadoop`.
+
+hadoop_conf_dir::
+ This is the configuration directory of the Yarn/MR2 service,
+ typically `/etc/hadoop/conf`.
+
Beeswax Configuration
~~~~~~~~~~~~~~~~~~~~~
View
4 tools/jenkins/build-functions
@@ -35,7 +35,8 @@ build_hadoop() {
fi
HADOOP_DIR=$HUE_ROOT/ext/hadoop
- export HADOOP_HOME="$HADOOP_DIR/${CDH_VERSION}"
+ export HADOOP_HDFS_HOME="$HADOOP_DIR/${CDH_VERSION}"
+ export HADOOP_BIN=$HADOOP_HDFS_HOME/bin/hadoop
mkdir -p $HADOOP_DIR
rm -rf $HADOOP_HOME
@@ -61,6 +62,7 @@ build_mr1() {
MR1_DIR=$HUE_ROOT/ext/mr1
export HADOOP_MR1_HOME="$MR1_DIR/${MR1_VERSION}"
+ export HADOOP_MR1_BIN="$HADOOP_MR1_HOME/bin/hadoop"
mkdir -p $MR1_DIR
rm -rf $HADOOP_MR1_HOME
View
267 tools/scripts/configure-hadoop.sh
@@ -1,267 +0,0 @@
-#!/bin/bash
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-
-if test -z $HADOOP_HOME ; then
- echo "Please define your HADOOP_HOME environment variable."
- exit
-fi
-
-SCRIPT_DIR=$(dirname $0)
-GIT_ROOT=$(cd $SCRIPT_DIR && pwd)/$(cd $SCRIPT_DIR && git rev-parse --show-cdup)
-BIND_IP=${BIND_IP:-localhost}
-HADOOP_TMP_DIR=$HADOOP_HOME/data
-
-if [ $(uname) == "Darwin" ]; then
- export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/1.6.0/Home
-fi
-SLAVE_IPS=${SLAVE_IPS:-127.0.0.1}
-
-echo -e ".======================== Parameters ========================\n"\
- "HADOOP_HOME : $HADOOP_HOME\n"\
- "GIT_ROOT : $GIT_ROOT\n"\
- "HADOOP_TMP_DIR : $HADOOP_TMP_DIR\n"\
- "============================================================\n"
-set -x
-
-
-# Configure a slave
-# Used by the sandboxer
-function write_hadoop_config() {
- SLAVE_IP=$1
- MASTER_IP=$2
- TARGET_DIR=$3
- TMP_DIR=$4
- if [ -z "$SLAVE_IP" -o -z "$MASTER_IP" -o -z "$TARGET_DIR" -o -z "$TMP_DIR" ]; then
- echo usage: $0 slave_ip master_ip target_conf_dir tmp_dir
- exit 1
- fi
-
- mkdir -p $TARGET_DIR
- cp $HADOOP_HOME/example-confs/conf.pseudo/hadoop-metrics.properties $TARGET_DIR
- cat > $TARGET_DIR/core-site.xml <<END
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
- <name>fs.default.name</name>
- <value>hdfs://${MASTER_IP}:8020</value>
-</property>
-<property>
- <name>hadoop.tmp.dir</name>
- <value>$TMP_DIR</value>
- <description>A base for other temporary directories.</description>
-</property>
-<property>
- <name>webinterface.private.actions</name>
- <value>true</value>
-</property>
-<property>
- <name>slave.host.name</name>
- <value>${SLAVE_IP}</value>
-</property>
-<property>
- <name>dfs.thrift.address</name>
- <value>${MASTER_IP}:10090</value>
-</property>
-<property>
- <name>jobtracker.thrift.address</name>
- <value>${MASTER_IP}:9290</value>
-</property>
-</configuration>
-END
- cat > $TARGET_DIR/hdfs-site.xml <<END
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
- <name>dfs.datanode.address</name>
- <value>${SLAVE_IP}:0</value>
-</property>
-<property>
- <name>dfs.datanode.ipc.address</name>
- <value>${SLAVE_IP}:0</value>
-</property>
-<property>
- <name>dfs.datanode.http.address</name>
- <value>${SLAVE_IP}:0</value>
-</property>
-<property>
- <name>dfs.http.address</name>
- <value>${MASTER_IP}:50070</value>
-</property>
-<property>
- <name>dfs.namenode.plugins</name>
- <value>org.apache.hadoop.thriftfs.NamenodePlugin</value>
- <description>Comma-separated list of namenode plug-ins to be activated.
- </description>
-</property>
-<property>
- <name>dfs.datanode.plugins</name>
- <value>org.apache.hadoop.thriftfs.DatanodePlugin</value>
- <description>Comma-separated list of datanode plug-ins to be activated.
- </description>
-</property>
-<!-- we dont really care about being super safe -->
-<property>
- <name>dfs.safemode.min.datanodes</name>
- <value>1</value>
-</property>
-<property>
- <name>dfs.safemode.extension</name>
- <value>5000</value>
- <description>
- Determines extension of safe mode in milliseconds
- after the threshold level is reached.
- </description>
-</property>
-</configuration>
-END
- cat > $TARGET_DIR/mapred-site.xml <<END
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
- <name>mapred.job.tracker</name>
- <value>${MASTER_IP}:8021</value>
-</property>
-<property>
- <name>mapred.job.tracker.http.address</name>
- <value>${MASTER_IP}:50030</value>
-</property>
-<property>
- <name>mapred.task.tracker.http.address</name>
- <value>${SLAVE_IP}:0</value>
-</property>
-<property>
- <name>mapred.jobtracker.plugins</name>
- <value>org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin</value>
- <description>Comma-separated list of jobtracker plug-ins to be activated.
- </description>
-</property>
-<property>
- <name>mapred.system.dir</name>
- <value>/hadoop/mapred/system</value>
-</property>
-<property>
- <name>mapred.local.dir</name>
- <value>$TMP_DIR/mapred/local</value>
-</property>
-</configuration>
-END
-
-}
-
-# Configure
-function configure() {
- perl -p -i -e "s,localhost,$BIND_IP,g" desktop/conf/pseudo-distributed.ini
- mkdir -p $HADOOP_TMP_DIR
- write_hadoop_config $BIND_IP $BIND_IP $HADOOP_HOME/conf $HADOOP_TMP_DIR
- idx=0
- for slave in $SLAVE_IPS ; do
- idx=$[$idx + 1]
- datadir=$HADOOP_TMP_DIR-slave-$idx
- write_hadoop_config $slave $BIND_IP $HADOOP_HOME/conf-slave-$idx $datadir
- done
-}
-
-function start() {
- pushd $HADOOP_HOME
- export HADOOP_CLASSPATH=$GIT_ROOT/desktop/libs/hadoop/java-lib/\*
- if [ ! -d $HADOOP_TMP_DIR/dfs/name ]; then
- bin/hadoop namenode -format
- fi
- # Pass HADOOP_OPTS=$JDB_ON to any hadoop-daemon.sh to enable jdb
- JDB_ON="-Xdebug -Xrunjdwp:transport=dt_socket,address=8901,server=y,suspend=n"
-
- HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-master" \
- bin/hadoop-daemon.sh start namenode
- idx=0
- for ip in $SLAVE_IPS ; do
- idx=$[$idx + 1]
- HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-slave-$idx" \
- HADOOP_CONF_DIR="$HADOOP_HOME/conf-slave-$idx" \
- HADOOP_LOG_DIR="$HADOOP_HOME/logs-slave-$idx" \
- bin/hadoop-daemon.sh start datanode || true
- done
- bin/hadoop dfsadmin -safemode wait
- HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-master" \
- bin/hadoop-daemon.sh start secondarynamenode
- HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-master" \
- bin/hadoop-daemon.sh start jobtracker
- idx=0
- for ip in $SLAVE_IPS ; do
- idx=$[$idx + 1]
- HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-slave-$idx" \
- HADOOP_CONF_DIR="$HADOOP_HOME/conf-slave-$idx" \
- HADOOP_LOG_DIR="$HADOOP_HOME/logs-slave-$idx" \
- bin/hadoop-daemon.sh start tasktracker || true
- done
- popd
-}
-
-function stop() {
- pushd $HADOOP_HOME
- for daemon in namenode jobtracker; do
- HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-master" \
- bin/hadoop-daemon.sh stop $daemon
- done
-
- idx=0
- for ip in $SLAVE_IPS ; do
- idx=$[$idx + 1]
- HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-slave-$idx" \
- HADOOP_CONF_DIR="$HADOOP_HOME/conf-slave-$idx" \
- bin/hadoop-daemon.sh --config $HADOOP_HOME/conf-slave-$idx stop tasktracker || true
- HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-slave-$idx" \
- HADOOP_CONF_DIR="$HADOOP_HOME/conf-slave-$idx" \
- bin/hadoop-daemon.sh --config $HADOOP_HOME/conf-slave-$idx stop datanode || true
- done
-
- popd
-}
-
-
-pushd $GIT_ROOT
-
-if [ ! -d .git ]; then
- echo "This script moves into your git root, but this has failed."
- exit 1
-fi
-
-if [ $# -eq 0 ]; then
- echo "Usage: $0 (all|checkout|build|build_jobsub|build_plugins|configure|start)"
- exit 1
-fi
-
-ARG=$1
-shift 1
-case $ARG
-in
- configure|start|write_hadoop_config|stop)
- $ARG $@
- ;;
- all)
- configure
- start
- ;;
- *)
- echo "Unrecognized: $ARG"
- exit 1
- ;;
-esac
Please sign in to comment.
Something went wrong with that request. Please try again.