Permalink
Browse files

Updated to latest.

  • Loading branch information...
1 parent 7825058 commit d82610fd5883b903ad5bf1a5dca1f18274f28391 @aching aching committed Nov 8, 2012
Showing 917 changed files with 120,978 additions and 96,701 deletions.
View
@@ -72,6 +72,7 @@ if [ $# = 0 ]; then
echo " jmxget get JMX exported values from NameNode or DataNode."
echo " oiv apply the offline fsimage viewer to an fsimage"
echo " oev apply the offline edits viewer to an edits file"
+ echo " oid apply the offline fsimage decompressor to an fsimage"
echo " Use -help to see options"
echo " jobtracker run the MapReduce job Tracker node"
echo " pipes run a Pipes job"
@@ -123,6 +124,7 @@ fi
# CLASSPATH initially contains $HADOOP_CONF_DIR
JMX_OPTS=""
CLASSPATH="${HADOOP_CONF_DIR}"
+CLASSPATH=${CLASSPATH}:$HADOOP_CLASSPATH
CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar
# for developers, add Hadoop classes to CLASSPATH
@@ -175,12 +177,6 @@ for f in $HADOOP_HOME/build/hadoop-*-tools.jar; do
TOOL_PATH=${TOOL_PATH}:$f;
done
-# add user-specified CLASSPATH before corona so that a newer
-# corona jar can be specified to override the deployed one
-if [ "$HADOOP_CLASSPATH" != "" ]; then
- CLASSPATH=${CLASSPATH}:${HADOOP_CLASSPATH}
-fi
-
# CORONA_PATH for corona daemons
if [ -d "$HADOOP_HOME/build/contrib/corona/classes" ]; then
CORONA_PATH=${CORONA_PATH}:$HADOOP_HOME/build/contrib/corona/classes
@@ -198,6 +194,15 @@ for f in $HADOOP_HOME/contrib/corona/lib/*.jar; do
CORONA_LIB_PATH=${CORONA_LIB_PATH}:$f;
done
+# NOTIFIER_PATH for the namespace notifier server daemon
+if [ -d "$HADOOP_HOME/build/contrib/namespace-notifier/classes" ]; then
+ NOTIFIER_PATH=${NOTIFIER_PATH}:$HADOOP_HOME/build/contrib/namespace-notifier/classes
+fi
+
+for f in $HADOOP_HOME/contrib/namespace-notifier/*.jar; do
+ NOTIFIER_PATH=${NOTIFIER_PATH}:$f;
+done
+
# default log directory & file
if [ "$HADOOP_LOG_DIR" = "" ]; then
HADOOP_LOG_DIR="$HADOOP_HOME/logs"
@@ -243,6 +248,17 @@ elif [ "$COMMAND" = "avatarnode" ] ; then
elif [ "$COMMAND" = "secondarynamenode" ] ; then
CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_SECONDARYNAMENODE_OPTS"
+elif [ "$COMMAND" = "raidnode" ] ; then
+ CLASS='org.apache.hadoop.raid.RaidNode'
+ JMX_OPTS=$HADOOP_JMX_RAIDNODE_OPTS
+ HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS"
+ CLASSPATH=${CORONA_LIB_PATH}:${CLASSPATH}
+elif [ "$COMMAND" = "notifier" ] ; then
+ CLASS='org.apache.hadoop.hdfs.notifier.server.ServerCore'
+ if [ "$NOTIFIER_PATH" != "" ]; then
+ CLASSPATH=${CLASSPATH}:${NOTIFIER_PATH}
+ fi
+ JMX_OPTS="$JMX_OPTS $NOTIFIER_JMX_OPTS"
elif [ "$COMMAND" = "fsshellservice" ] ; then
CLASS='org.apache.hadoop.hdfs.fsshellservice.FsShellServiceImpl'
if [ -d "$HADOOP_HOME/build/contrib/corona/lib" ]; then
@@ -309,6 +325,9 @@ elif [ "$COMMAND" = "oiv" ] ; then
elif [ "$COMMAND" = "oev" ] ; then
CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "oid" ] ; then
+ CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageDecompressor
+ HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
elif [ "$COMMAND" = "jmxget" ] ; then
CLASS=org.apache.hadoop.hdfs.tools.JMXGet
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
@@ -324,17 +343,26 @@ elif [ "$COMMAND" = "coronaclustermanager" ] ; then
JMX_OPTS=$HADOOP_JMX_CORONACLUSTERMANAGER_OPTS
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_CORONACLUSTERMANAGER_OPTS"
# Corona lib path should be first to ensure that it uses the right thrift JAR
- CLASSPATH=${CORONA_LIB_PATH}:${CLASSPATH}
+ CLASSPATH=${CORONA_LIB_PATH}:${CLUSTER_MANAGER_LIB_PATH}:${CLASSPATH}
elif [ "$COMMAND" = "coronatasktracker" ] ; then
CLASS=org.apache.hadoop.mapred.CoronaTaskTracker
JMX_OPTS=$HADOOP_JMX_TASKTRACKER_OPTS
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_TASKTRACKER_OPTS"
+ # For corona task trackers, the tasks should not get the thrift library.
+ MAPREDUCE_TASK_SYSTEM_CLASSPATH=${CLASSPATH}
+ export MAPREDUCE_TASK_SYSTEM_CLASSPATH
# See coronaclustermanager comment
CLASSPATH=${CORONA_LIB_PATH}:${CLASSPATH}
elif [ "$COMMAND" = "coronaproxyjobtracker" ] ; then
CLASS=org.apache.hadoop.mapred.ProxyJobTracker
JMX_OPTS=$HADOOP_JMX_CORONAPROXYJOBTRACKER_OPTS
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_CORONAPROXYJOBTRACKER_OPTS"
+ # See coronaclustermanager comment
+ CLASSPATH=${CORONA_LIB_PATH}:${CLASSPATH}
+elif [ "$COMMAND" = "coronaclient" ] ; then
+ CLASS=org.apache.hadoop.corona.CoronaClient
+ HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+ CLASSPATH=${CORONA_LIB_PATH}:${CLASSPATH}
elif [ "$COMMAND" = "coronaadmin" ] ; then
CLASS=org.apache.hadoop.corona.CoronaAdmin
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
@@ -350,9 +378,10 @@ elif [ "$COMMAND" = "tasktracker" ] ; then
elif [ "$COMMAND" = "multitasktracker" ] ; then
CLASS=org.apache.hadoop.mapred.MultiTaskTracker
HADOOP_ROOT_LOGGER=${HADOOP_TASKTRACKER_LOGGER:-$HADOOP_ROOT_LOGGER}
+ HADOOP_OPTS="$HADOOP_OPTS $HADOOP_MULTITASKTRACKER_OPTS"
# This should be the number of tasktrackers
- if [ -n "$MULTI_TT_SIZE" ] ; then
- CMDLINE_OPTS="$MULTI_TT_SIZE"
+ if [ -n "$MULTI_TT_OPTIONS" ] ; then
+ CMDLINE_OPTS="$MULTI_TT_OPTIONS"
else
CMDLINE_OPTS="1"
fi
@@ -374,7 +403,7 @@ elif [ "$COMMAND" = "fastcopy" ] ; then
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
elif [ "$COMMAND" = "distcp" ] ; then
CLASS=org.apache.hadoop.tools.DistCp
- CLASSPATH=${CLASSPATH}:${TOOL_PATH}
+ CLASSPATH=${CORONA_LIB_PATH}:${CLASSPATH}:${TOOL_PATH}
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
elif [ "$COMMAND" = "daemonlog" ] ; then
CLASS=org.apache.hadoop.log.LogLevel
@@ -463,4 +492,5 @@ if [ "$HADOOP_DEPLOYMENT" == "server" ]; then
fi
# run it
-exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $JMX_OPTS -classpath "$CLASSPATH" $CLASS $CMDLINE_OPTS "$@"
+export CLASSPATH
+exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $JMX_OPTS "-Dfb_hadoop_version=0.20" $CLASS $CMDLINE_OPTS "$@"
View
@@ -24,7 +24,8 @@ bin=`cd "$bin"; pwd`
. "$bin"/hadoop-config.sh
# start corona daemons
-# start clustermanager first to minimize connection errors at startup
-"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR start coronaclustermanager
+# start start-proxyjt.sh first so that clustermanager can be started correctly
"$bin"/start-proxyjt.sh --config $HADOOP_CONF_DIR
+sleep 1
+"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR start coronaclustermanager
"$bin"/hadoop-daemons.sh --config $HADOOP_CONF_DIR start coronatasktracker
View
0 bin/start-fsshellservice.sh 100755 → 100644
No changes.
@@ -30,13 +30,7 @@ for f in "$bin"/../build/contrib/*/*.jar; do
export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:$f;
done
-export HADOOP_OPTS="$HADOOP_OPTS -Dcom.sun.management.jmxremote
--Dcom.sun.management.jmxremote.port=8697 \
--Dcom.sun.management.jmxremote.authenticate=false
--Dcom.sun.management.jmxremote.ssl=false \
--verbose:gc -XX:+PrintGCDateStamps -XX:+PrintGCDetails \
--Xloggc:/usr/local/hadoop/logs/MRSIM/multitasktracker.gc.log \
--XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC"
+export HADOOP_MULTITASKTRACKER_OPTS=" -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=8697 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -verbose:gc -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:/usr/local/hadoop/logs/MRSIM/multitasktracker.gc.log -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC"
# start mapred daemons
# start jobtracker first to minimize connection errors at startup
-"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR start multitasktracker
+"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR start multitasktracker
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+usage="Usage: start-namespace-notifier.sh"
+
+params=$#
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/hadoop-config.sh
+. "$bin"/../conf/hadoop-env.sh
+
+# get arguments
+if [ $# -ge 1 ]; then
+ echo $usage
+fi
+
+export NOTIFIER_JMX_OPTS=" -Dcom.sun.management.jmxremote.port=$NOTIFIER_JMX_PORT -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
+
+"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR start notifier
View
@@ -37,4 +37,4 @@ if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then
fi
export HADOOP_DAEMON_OPTS=$HADOOP_RAIDNODE_OPTS
-"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR start org.apache.hadoop.raid.RaidNode
+"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR start raidnode
View
0 bin/stop-fsshellservice.sh 100755 → 100644
No changes.
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+usage="Usage: stop-namespace-notifier.sh"
+
+params=$#
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/hadoop-config.sh
+
+# get arguments
+if [ $# -ge 1 ]; then
+ echo $usage
+fi
+
+"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR stop notifier
View
@@ -107,7 +107,7 @@
<property name="test.junit.fork.mode" value="perTest" />
<property name="test.junit.printsummary" value="yes" />
<property name="test.junit.haltonfailure" value="no" />
- <property name="test.junit.maxmemory" value="512m" />
+ <property name="test.junit.maxmemory" value="1024m" />
<property name="test.tools.input.dir" value="${basedir}/src/test/tools/data"/>
<!-- The default user.home might have lots of dirs that will slow down
tets that scan user.home, so allow a way to specify a custom,
@@ -229,6 +229,14 @@
</and>
</condition>
+ <!-- Indicate is Snappy native library should be bundled with Hadoop or not -->
+ <property name="bundle.snappy" value="true"/>
+
+ <!-- Snappy native library location -->
+ <property name="snappy.prefix" value="${native.src.dir}/lib/snappy"/>
+ <property name="snappy.lib" value="${snappy.prefix}/lib"/>
+ <property name="snappy.include" value="${snappy.prefix}/include"/>
+
<!-- the normal classpath -->
<path id="classpath">
<pathelement location="${build.classes}"/>
@@ -499,6 +507,7 @@
<mkdir dir="${build.native}/lib"/>
<mkdir dir="${build.native}/src/org/apache/hadoop/io/compress/zlib"/>
+ <mkdir dir="${build.native}/src/org/apache/hadoop/io/compress/snappy"/>
<mkdir dir="${build.native}/src/org/apache/hadoop/io/compress/lzma"/>
<mkdir dir="${build.native}/src/org/apache/hadoop/syscall"/>
<mkdir dir="${build.native}/src/org/apache/hadoop/util"/>
@@ -559,8 +568,20 @@
<env key="OS_ARCH" value="${os.arch}"/>
<env key="JVM_DATA_MODEL" value="${sun.arch.data.model}"/>
<env key="HADOOP_NATIVE_SRCDIR" value="${native.src.dir}"/>
- <arg line="${native.src.dir}/configure LDFLAGS='-L${basedir}/nativelib/lzma' CPPFLAGS='-I${basedir}/nativelib/lzma'"/>
- </exec>
+ <arg line="${native.src.dir}/configure
+ LDFLAGS='-L${snappy.lib} -L${basedir}/nativelib/lzma -lrt'
+ CPPFLAGS='-I${snappy.include} -I${basedir}/nativelib/lzma'"/>
+ </exec>
+
+ <javah
+ classpath="${build.classes}"
+ destdir="${build.native}/src/org/apache/hadoop/io/compress/snappy"
+ force="yes"
+ verbose="yes"
+ >
+ <class name="org.apache.hadoop.io.compress.snappy.SnappyCompressor"/>
+ <class name="org.apache.hadoop.io.compress.snappy.SnappyDecompressor"/>
+ </javah>
<exec dir="${build.native}" executable="${make.cmd}" failonerror="true">
<env key="OS_NAME" value="${os.name}"/>
@@ -569,8 +590,16 @@
<env key="HADOOP_NATIVE_SRCDIR" value="${native.src.dir}"/>
</exec>
- <exec dir="${build.native}" executable="sh" failonerror="true">
- <arg line="${build.native}/libtool --mode=install cp ${build.native}/lib/libhadoop.la ${build.native}/lib"/>
+ <delete>
+ <fileset dir="${build.native}/lib" includes="libsnappy.*"/>
+ </delete>
+ <copy todir="${build.native}/lib">
+ <fileset dir="${snappy.lib}" includes="libsnappy.*"/>
+ <fileset dir="${build.native}/src/org/apache/hadoop/io/compress/snappy/.libs/" includes="libhadoopsnappy.*"/>
+ </copy>
+
+ <exec dir="${build.native}" executable="sh" failonerror="true">
+ <arg line="${build.native}/libtool --mode=install cp ${build.native}/lib/libhadoop.la ${build.native}/lib"/>
</exec>
<delete>
<fileset dir="${build.native}/lib" includes="liblzma.so*"/>
@@ -763,6 +792,7 @@
<mkdir dir="${test.debug.data}"/>
<copy file="${test.src.dir}/org/apache/hadoop/mapred/testscript.txt" todir="${test.debug.data}"/>
<copy file="${test.src.dir}/org/apache/hadoop/mapred/test.txt" todir="${test.cache.data}"/>
+ <copy file="${test.src.dir}/org/apache/hadoop/io/compress/snappy/testsnappy.txt" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/org/apache/hadoop/mapred/test.jar" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/org/apache/hadoop/mapred/test.zip" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/org/apache/hadoop/mapred/test.tar" todir="${test.cache.data}"/>
@@ -842,7 +872,7 @@
<sysproperty key="test.build.extraconf" value="${test.build.extraconf}" />
<sysproperty key="hadoop.policy.file" value="hadoop-policy.xml"/>
<sysproperty key="java.library.path"
- value="${build.native}/lib:${lib.dir}/native/${build.platform}"/>
+ value="${build.native}/lib:${lib.dir}/native/${build.platform}:${snappy.lib}"/>
<sysproperty key="install.c++.examples" value="${install.c++.examples}"/>
<sysproperty key="user.home" value="${test.user.home}"/>
<env key="LD_LIBRARY_PATH" value="${build.native}/lib${path.separator}${env.LD_LIBRARY_PATH}"/>
@@ -905,7 +935,7 @@
</subant>
</target>
- <target name="test" depends="clean, compile" description="Run core, contrib unit tests">
+ <target name="test" depends="clean, compile, compile-native" description="Run core, contrib unit tests">
<subant target="test-core" failonerror="false">
<fileset file="build.xml"/>
</subant>
@@ -1214,6 +1244,8 @@
<env key="BASE_NATIVE_LIB_DIR" value="${lib.dir}/native"/>
<env key="BUILD_NATIVE_DIR" value="${build.dir}/native"/>
<env key="DIST_LIB_DIR" value="${dist.dir}/lib/native"/>
+ <env key="BUNDLE_SNAPPY_LIB" value="${bundle.snappy}"/>
+ <env key="SNAPPY_LIB_DIR" value="${snappy.prefix}/lib"/>
<arg line="${native.src.dir}/packageNativeHadoop.sh"/>
</exec>
@@ -1306,6 +1338,17 @@
</macro_tar>
</target>
+ <target name="superjar" depends="bin-package">
+ <jar jarfile="${build.dir}/${final.name}-super.jar">
+ <zipgroupfileset dir="lib" includes="*jar"/>
+ <zipgroupfileset dir="build" includes="*jar"/>
+ <zipgroupfileset dir="build/contrib/corona" includes="*jar"/>
+ <zipgroupfileset dir="build/contrib/corona/lib" includes="*jar"/>
+ <zipgroupfileset dir="build/ivy/lib/Hadoop/common/" includes="*jar"/>
+ <zipgroupfileset dir="build/ivy/lib/corona/common/" includes="*jar"/>
+ </jar>
+ </target>
+
<target name="bin-package" depends="compile, jar, examples, tools-jar, jar-test, ant-tasks, package-librecordio"
description="assembles artifacts for binary target">
<mkdir dir="${dist.dir}"/>
@@ -1327,6 +1370,8 @@
<env key="BASE_NATIVE_LIB_DIR" value="${lib.dir}/native"/>
<env key="BUILD_NATIVE_DIR" value="${build.dir}/native"/>
<env key="DIST_LIB_DIR" value="${dist.dir}/lib/native"/>
+ <env key="BUNDLE_SNAPPY_LIB" value="${bundle.snappy}"/>
+ <env key="SNAPPY_LIB_DIR" value="${snappy.prefix}/lib"/>
<arg line="${native.src.dir}/packageNativeHadoop.sh"/>
</exec>
@@ -2013,8 +2058,8 @@
</target>
<!-- end of task-controller target -->
- <target name="print-version">
- <echo message="${version}" />
+ <target name="print-version">
+ <echo message="${version}" />
</target>
</project>
Oops, something went wrong.

0 comments on commit d82610f

Please sign in to comment.