Refresh up to hive99-r19817-11012011

Summary:

Task ID: #

Blame Rev:

Reviewers:

CC:

Test Plan:

Revert Plan:

FB-CHANGES.txt

@@ -230,4 +230,3 @@ Release 0.20.3 + FB - Unreleased.
     HDFS-955        Fix Edits log/Save FSImage bugs
     HADOOP-6683     the first optimization: ZlibCompressor does not fully utilize the buffer
     HADOOP-7111     Several TFile tests failing when native libraries are present
-    HDFS-2139       Fast copy for HDFS.

bin/hadoop

@@ -62,7 +62,6 @@ if [ $# = 0 ]; then
   echo "  dfsadmin             run a DFS admin client"
   echo "  mradmin              run a Map-Reduce admin client"
   echo "  fsck                 run a DFS filesystem checking utility"
-  echo "  avatarfsck           run a avatar DFS filesystem checking utility"
   echo "  raidfsck [path]      run RAID-aware filesystem checking utility"
   echo "  raidshell [options]  run RAID-shell utility"
   echo "  fs                   run a generic filesystem user client"
@@ -119,6 +118,7 @@ if [ "$HADOOP_HEAPSIZE" != "" ]; then
 fi
 
 # CLASSPATH initially contains $HADOOP_CONF_DIR
+JMX_OPTS=""
 CLASSPATH="${HADOOP_CONF_DIR}"
 CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar
 
@@ -155,6 +155,10 @@ for f in $HADOOP_HOME/lib/*.jar; do
   CLASSPATH=${CLASSPATH}:$f;
 done
 
+for f in $HADOOP_HOME/lib/logger/*.jar; do
+  CLASSPATH=${CLASSPATH}:$f;
+done
+
 if [ -d "$HADOOP_HOME/build/ivy/lib/Hadoop/common" ]; then
 for f in $HADOOP_HOME/build/ivy/lib/Hadoop/common/*.jar; do
   CLASSPATH=${CLASSPATH}:$f;
@@ -221,6 +225,7 @@ HADOOP_OPTS="$HADOOP_OPTS -ea:org.apache.hadoop.mapred..."
 # figure out which class to run
 if [ "$COMMAND" = "namenode" ] ; then
   CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode'
+  JMX_OPTS=$HADOOP_JMX_NAMENODE_OPTS
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_NAMENODE_OPTS"
 elif [ "$COMMAND" = "avatarshell" ] ; then
   CLASS='org.apache.hadoop.hdfs.AvatarShell'
@@ -230,16 +235,19 @@ elif [ "$COMMAND" = "avatarzk" ] ; then
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_CLIENT_OPTS"
 elif [ "$COMMAND" = "avatarnode" ] ; then
   CLASS='org.apache.hadoop.hdfs.server.namenode.AvatarNode'
+  JMX_OPTS=$HADOOP_JMX_NAMENODE_OPTS
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_NAMENODE_OPTS"
 elif [ "$COMMAND" = "secondarynamenode" ] ; then
   CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_SECONDARYNAMENODE_OPTS"
 elif [ "$COMMAND" = "avatardatanode" ] ; then
   CLASS='org.apache.hadoop.hdfs.server.datanode.AvatarDataNode'
+  JMX_OPTS=$HADOOP_JMX_DATANODE_OPTS
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_DATANODE_OPTS"
   HADOOP_ROOT_LOGGER=${HADOOP_DATANODE_LOGGER:-$HADOOP_ROOT_LOGGER}
 elif [ "$COMMAND" = "datanode" ] ; then
   CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode'
+  JMX_OPTS=$HADOOP_JMX_DATANODE_OPTS
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_DATANODE_OPTS"
   HADOOP_ROOT_LOGGER=${HADOOP_DATANODE_LOGGER:-$HADOOP_ROOT_LOGGER}
 elif [ "$COMMAND" = "fs" ] ; then
@@ -260,9 +268,6 @@ elif [ "$COMMAND" = "version" ] ; then
 elif [ "$COMMAND" = "fsck" ] ; then
   CLASS=org.apache.hadoop.hdfs.tools.DFSck
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-elif [ "$COMMAND" = "avatarfsck" ] ; then
-  CLASS=org.apache.hadoop.hdfs.tools.AvatarDFSck
-  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
 elif [ "$COMMAND" = "raidfsck" ] ; then
   CLASS=org.apache.hadoop.raid.RaidShell
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
@@ -285,25 +290,28 @@ elif [ "$COMMAND" = "jmxget" ] ; then
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
 elif [ "$COMMAND" = "jobtracker" ] ; then
   CLASS=org.apache.hadoop.mapred.JobTracker
+  JMX_OPTS=$HADOOP_JMX_JOBTRACKER_OPTS
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_JOBTRACKER_OPTS"
   if [ -n "$HADOOP_INSTANCE" ] ; then
     CMDLINE_OPTS="-instance $HADOOP_INSTANCE $CMDLINE_OPTS"
   fi
 elif [ "$COMMAND" = "coronaclustermanager" ] ; then
   CLASS=org.apache.hadoop.corona.ClusterManagerServer
+  JMX_OPTS=$HADOOP_JMX_CORONACLUSTERMANAGER_OPTS
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_CORONACLUSTERMANAGER_OPTS"
-  # Corona lib path should be first to ensure that it uses the right thrift JAR
-  CLASSPATH=${CORONA_LIB_PATH}:${CLASSPATH}
+  CLASSPATH=${CLASSPATH}:${CORONA_LIB_PATH}
 elif [ "$COMMAND" = "coronatasktracker" ] ; then
   CLASS=org.apache.hadoop.mapred.CoronaTaskTracker
+  JMX_OPTS=$HADOOP_JMX_TASKTRACKER_OPTS
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_TASKTRACKER_OPTS"
-  # See coronaclustermanager comment
-  CLASSPATH=${CORONA_LIB_PATH}:${CLASSPATH}
+  CLASSPATH=${CLASSPATH}:${CORONA_LIB_PATH}
 elif [ "$COMMAND" = "coronaproxyjobtracker" ] ; then
   CLASS=org.apache.hadoop.mapred.ProxyJobTracker
+  JMX_OPTS=$HADOOP_JMX_CORONAPROXYJOBTRACKER_OPTS
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_CORONAPROXYJOBTRACKER_OPTS"
 elif [ "$COMMAND" = "tasktracker" ] ; then
   CLASS=org.apache.hadoop.mapred.TaskTracker
+  JMX_OPTS=$HADOOP_JMX_TASKTRACKER_OPTS
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_GC_LOG_OPTS $HADOOP_TASKTRACKER_OPTS"
   HADOOP_ROOT_LOGGER=${HADOOP_TASKTRACKER_LOGGER:-$HADOOP_ROOT_LOGGER}
   if [ -n "$HADOOP_INSTANCE" ] ; then
@@ -416,4 +424,4 @@ if [ "$HADOOP_DEPLOYMENT" == "server" ]; then
 fi
 
 # run it
-exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS -classpath "$CLASSPATH" $CLASS $CMDLINE_OPTS "$@"
+exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $JMX_OPTS -classpath "$CLASSPATH" $CLASS $CMDLINE_OPTS "$@"

bin/hadoop-config.sh

@@ -74,16 +74,23 @@ then
     then
         shift
         instance=$1
-        if [ "$instance" != "0" && "$instance" != "1" ]
-        then
-            echo "Instance must be 0 or 1"
-            exit -1
-        fi
         shift
         export HADOOP_INSTANCE=$instance
     fi
 fi
 
+#set the namenode jmx port if namenode_jmx is given
+if [ $# -gt 1 ]
+then
+    if [ "--namenode_jmx" = "$1" ]
+    then
+        shift
+        namenode_jmx=$1
+        shift
+        export HADOOP_JMX_NAMENODE_OPTS="-Dcom.sun.management.jmxremote.port=$namenode_jmx"
+    fi
+fi
+
 # YourKit Java Profiling
 # Note that you need to have yjpagent.so & yjp.jar on your computer and have
 # LD_LIBRARY_PATH entry to their dir location. for example:

bin/hadoop-daemon.sh

@@ -28,7 +28,7 @@
 #   HADOOP_NICENESS The scheduling priority for daemons. Defaults to 0.
 ##
 
-usage="Usage: hadoop-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] [--instance <0|1>] (start|stop) <hadoop-command> <args...>"
+usage="Usage: hadoop-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] [--instance uniqid] [--namenode_jmx <port>] (start|stop) <hadoop-command> <args...>"
 
 # if no args specified, show usage
 if [ $# -le 1 ]; then

bin/stop-tasktracker.sh

@@ -43,7 +43,7 @@ if [ -f $pid ]; then
   pidvalue=$(cat $pid)
   for i in `ps -o pid --no-headers --ppid $pidvalue`; do
     echo "Killing process group $i"
-    kill -s 9 -- -$i;
+    kill -- -$i;
   done
 fi
 

build.xml

@@ -494,8 +494,6 @@
     <mkdir dir="${build.native}/src/org/apache/hadoop/io/compress/zlib"/>
     <mkdir dir="${build.native}/src/org/apache/hadoop/io/compress/lzma"/>
     <mkdir dir="${build.native}/src/org/apache/hadoop/syscall"/>
-    <mkdir dir="${build.native}/src/org/apache/hadoop/util"/>
-    <mkdir dir="${build.native}/src/org/apache/hadoop/io/nativeio"/>
 
     <javah
       classpath="${build.classes}"
@@ -517,17 +515,6 @@
       <class name="org.apache.hadoop.io.compress.lzma.LzmaDecompressor" />
     </javah>
 
-    <echo> Running javah NativeIO.java  </echo>
-    <javah
-      classpath="${build.classes}"
-      destdir="${build.native}/src/org/apache/hadoop/io/nativeio"
-      force="yes"
-      verbose="yes"
-      >
-      <class name="org.apache.hadoop.io.nativeio.NativeIO" />
-    </javah>
-
-    <echo> Running javah LinuxSystemCall.java  </echo>
     <javah
       classpath="${build.classes}"
       destdir="${build.native}/src/org/apache/hadoop/syscall"
@@ -537,16 +524,6 @@
       <class name="org.apache.hadoop.syscall.LinuxSystemCall" />
     </javah>
 
-    <echo> Running javah bulk_crc32.java  </echo>
-    <javah
-      classpath="${build.classes}"
-      destdir="${build.native}/src/org/apache/hadoop/util"
-      force="yes"
-      verbose="yes"
-      >
-      <class name="org.apache.hadoop.util.NativeCrc32" />
-    </javah>
-
     <exec dir="${build.native}" executable="sh" failonerror="true">
       <env key="OS_NAME" value="${os.name}"/>
       <env key="OS_ARCH" value="${os.arch}"/>
@@ -1249,7 +1226,6 @@
 
     <copy todir="${dist.dir}/" file="build.xml"/>
 
-    <chmod perm="ugo+x" file="${dist.dir}/src/native/configure"/>
     <chmod perm="ugo+x" type="file" parallel="false">
         <fileset dir="${dist.dir}/bin"/>
         <fileset dir="${dist.dir}/src/contrib/">
@@ -1274,15 +1250,13 @@
           <exclude name="${final.name}/contrib/*/bin/*" />
           <exclude name="${final.name}/src/contrib/ec2/bin/*" />
           <exclude name="${final.name}/src/contrib/ec2/bin/image/*" />
-          <exclude name="${final.name}/src/native/configure" />
           <include name="${final.name}/**" />
         </tarfileset>
         <tarfileset dir="${build.dir}" mode="755">
           <include name="${final.name}/bin/*" />
           <include name="${final.name}/contrib/*/bin/*" />
           <include name="${final.name}/src/contrib/ec2/bin/*" />
           <include name="${final.name}/src/contrib/ec2/bin/image/*" />
-          <include name="${final.name}/src/native/configure" />
         </tarfileset>
       </param.listofitems>
     </macro_tar>