From 9b7b98e8f33836334990719479cbc762de39cfae Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 6 Jun 2012 20:17:23 +0000 Subject: [PATCH 01/91] svn merge -c -1346491 for re-committing HADOOP-8368. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1347092 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + hadoop-common-project/hadoop-common/pom.xml | 103 ++----- .../hadoop-common/src/CMakeLists.txt | 131 +++++++++ .../hadoop-common/src/config.h.cmake | 10 + .../src/main/native/.autom4te.cfg | 42 --- .../hadoop-common/src/main/native/Makefile.am | 66 ----- .../src/main/native/acinclude.m4 | 28 -- .../src/main/native/configure.ac | 130 --------- .../src/main/native/lib/Makefile.am | 47 --- .../hadoop/io/compress/lz4/Lz4Compressor.c | 5 +- .../hadoop/io/compress/lz4/Lz4Decompressor.c | 5 +- .../io/compress/snappy/SnappyCompressor.c | 36 +-- .../io/compress/snappy/SnappyDecompressor.c | 36 +-- .../org_apache_hadoop_io_compress_snappy.h | 41 +-- .../hadoop/io/compress/zlib/Makefile.am | 53 ---- .../hadoop/io/compress/zlib/ZlibCompressor.c | 32 +-- .../io/compress/zlib/ZlibDecompressor.c | 32 +-- .../zlib/org_apache_hadoop_io_compress_zlib.h | 39 +-- .../org/apache/hadoop/io/nativeio/NativeIO.c | 4 +- .../src/org/apache/hadoop/util/NativeCrc32.c | 4 +- .../src/main/native/src/org_apache_hadoop.h | 17 +- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 76 +---- .../hadoop-hdfs/src/CMakeLists.txt | 123 ++++++++ .../hadoop-hdfs/src/config.h.cmake | 6 + .../src/contrib/fuse-dfs/Makefile.am | 27 -- .../src/contrib/fuse-dfs/acinclude.m4 | 270 ------------------ .../src/contrib/fuse-dfs/configure.ac | 82 ------ .../hadoop-hdfs/src/contrib/fuse-dfs/pom.xml | 164 ----------- .../src/contrib/fuse-dfs/src/CMakeLists.txt | 73 +++++ .../src/contrib/fuse-dfs/src/Makefile.am | 22 -- .../src/contrib/fuse-dfs/src/fuse_dfs.h | 8 +- .../hadoop-hdfs/src/main/native/Makefile.am | 42 --- .../hadoop-hdfs/src/main/native/configure.ac | 125 -------- .../src/main/native/m4/apfunctions.m4 | 41 --- .../hadoop-hdfs/src/main/native/m4/apjava.m4 | 142 --------- .../src/main/native/m4/apsupport.m4 | 168 ----------- hadoop-hdfs-project/pom.xml | 1 - .../hadoop-yarn-server-nodemanager/pom.xml | 64 ++--- .../src/CMakeLists.txt | 66 +++++ .../src/config.h.cmake | 6 + .../native/container-executor/.autom4te.cfg | 42 --- .../.deps/container-executor.Po | 1 - .../native/container-executor/Makefile.am | 32 --- .../native/container-executor/configure.ac | 54 ---- .../native/container-executor/impl/main.c | 5 +- 45 files changed, 512 insertions(+), 1991 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/CMakeLists.txt create mode 100644 hadoop-common-project/hadoop-common/src/config.h.cmake delete mode 100644 hadoop-common-project/hadoop-common/src/main/native/.autom4te.cfg delete mode 100644 hadoop-common-project/hadoop-common/src/main/native/Makefile.am delete mode 100644 hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 delete mode 100644 hadoop-common-project/hadoop-common/src/main/native/configure.ac delete mode 100644 hadoop-common-project/hadoop-common/src/main/native/lib/Makefile.am delete mode 100644 hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/config.h.cmake delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/Makefile.am delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/acinclude.m4 delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/configure.ac delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/pom.xml create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/CMakeLists.txt delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/Makefile.am delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/Makefile.am delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/configure.ac delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apfunctions.m4 delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apjava.m4 delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apsupport.m4 create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake delete mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.autom4te.cfg delete mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.deps/container-executor.Po delete mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/Makefile.am delete mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/configure.ac diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 259b5a428d..57c387ac7d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -226,6 +226,8 @@ Release 2.0.1-alpha - UNRELEASED HADOOP-8358. Config-related WARN for dfs.web.ugi can be avoided. (harsh) + HADOOP-8368. Use CMake rather than autotools to build native code (ccccabe via tucu) + HADOOP-8450. Remove src/test/system. (eli) BUG FIXES diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index a36b74d1e0..c045a7fcdb 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -536,31 +536,10 @@ /usr/local ${snappy.prefix}/lib ${snappy.prefix}/include + - - org.apache.maven.plugins - maven-antrun-plugin - - - compile - compile - - run - - - - - - - - - - - - - org.codehaus.mojo native-maven-plugin @@ -590,73 +569,27 @@ - org.codehaus.mojo - make-maven-plugin + org.apache.maven.plugins + maven-antrun-plugin - compile + make compile - - autoreconf - configure - make-install - + run + + + + + + + + + + + + - - - ${project.build.directory}/native - - -i - -f - - - - - - OS_NAME - ${os.name} - - - OS_ARCH - ${os.arch} - - - JVM_DATA_MODEL - ${sun.arch.data.model} - - - - CPPFLAGS=-I${snappy.include} - LDFLAGS=-L${snappy.lib} - - ${project.build.directory}/native - /usr/local - - - - - OS_NAME - ${os.name} - - - OS_ARCH - ${os.arch} - - - JVM_DATA_MODEL - ${sun.arch.data.model} - - - HADOOP_NATIVE_SRCDIR - ${project.build.directory}/native - - - - - ${project.build.directory}/native/target - - @@ -700,7 +633,7 @@ maven-antrun-plugin - compile + kdc compile run diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt new file mode 100644 index 0000000000..127e2d9c20 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -0,0 +1,131 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cmake_minimum_required(VERSION 2.6 FATAL_ERROR) + +# Default to release builds +set(CMAKE_BUILD_TYPE, Release) + +# If JVM_ARCH_DATA_MODEL is 32, compile all binaries as 32-bit. +# This variable is set by maven. +if (JVM_ARCH_DATA_MODEL EQUAL 32) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") + set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32") +endif (JVM_ARCH_DATA_MODEL EQUAL 32) + +# Compile a library with both shared and static variants +function(add_dual_library LIBNAME) + add_library(${LIBNAME} SHARED ${ARGN}) + add_library(${LIBNAME}_static STATIC ${ARGN}) + set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME}) +endfunction(add_dual_library) + +# Link both a static and a dynamic target against some libraries +function(target_link_dual_libraries LIBNAME) + target_link_libraries(${LIBNAME} ${ARGN}) + target_link_libraries(${LIBNAME}_static ${ARGN}) +endfunction(target_link_dual_libraries) + +function(output_directory TGT DIR) + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") +endfunction(output_directory TGT DIR) + +function(dual_output_directory TGT DIR) + output_directory(${TGT} "${DIR}") + output_directory(${TGT}_static "${DIR}") +endfunction(dual_output_directory TGT DIR) + +if (NOT GENERATED_JAVAH) + # Must identify where the generated headers have been placed + MESSAGE(FATAL_ERROR "You must set the cmake variable GENERATED_JAVAH") +endif (NOT GENERATED_JAVAH) +find_package(JNI REQUIRED) +find_package(ZLIB REQUIRED) + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64") +set(D main/native/src/org/apache/hadoop) + +GET_FILENAME_COMPONENT(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME) + +INCLUDE(CheckFunctionExists) +INCLUDE(CheckCSourceCompiles) +CHECK_FUNCTION_EXISTS(sync_file_range HAVE_SYNC_FILE_RANGE) +CHECK_FUNCTION_EXISTS(posix_fadvise HAVE_POSIX_FADVISE) + +find_library(SNAPPY_LIBRARY NAMES snappy PATHS) +find_path(SNAPPY_INCLUDE_DIR NAMES snappy.h PATHS) +if (SNAPPY_LIBRARY) + GET_FILENAME_COMPONENT(HADOOP_SNAPPY_LIBRARY ${SNAPPY_LIBRARY} NAME) + set(SNAPPY_SOURCE_FILES + "${D}/io/compress/snappy/SnappyCompressor.c" + "${D}/io/compress/snappy/SnappyDecompressor.c") +else (${SNAPPY_LIBRARY}) + set(SNAPPY_INCLUDE_DIR "") + set(SNAPPY_SOURCE_FILES "") +endif (SNAPPY_LIBRARY) + +include_directories( + ${GENERATED_JAVAH} + main/native/src + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_BINARY_DIR} + ${JNI_INCLUDE_DIRS} + ${ZLIB_INCLUDE_DIRS} + ${SNAPPY_INCLUDE_DIR} +) +CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h) + +add_dual_library(hadoop + ${D}/io/compress/lz4/Lz4Compressor.c + ${D}/io/compress/lz4/Lz4Decompressor.c + ${D}/io/compress/lz4/lz4.c + ${SNAPPY_SOURCE_FILES} + ${D}/io/compress/zlib/ZlibCompressor.c + ${D}/io/compress/zlib/ZlibDecompressor.c + ${D}/io/nativeio/NativeIO.c + ${D}/io/nativeio/errno_enum.c + ${D}/io/nativeio/file_descriptor.c + ${D}/security/JniBasedUnixGroupsMapping.c + ${D}/security/JniBasedUnixGroupsNetgroupMapping.c + ${D}/security/getGroup.c + ${D}/util/NativeCrc32.c + ${D}/util/bulk_crc32.c +) +target_link_dual_libraries(hadoop + dl + ${JAVA_JVM_LIBRARY} +) +SET(LIBHADOOP_VERSION "1.0.0") +SET_TARGET_PROPERTIES(hadoop PROPERTIES + SOVERSION ${LIBHADOOP_VERSION}) +dual_output_directory(hadoop target/usr/local/lib) + +if (HADOOP_RUNAS_HOME) + add_executable(runAs + test/system/c++/runAs/runAs.c + test/system/c++/runAs/main.c + ) + output_directory(runAs target/usr/local/bin) +endif (HADOOP_RUNAS_HOME) diff --git a/hadoop-common-project/hadoop-common/src/config.h.cmake b/hadoop-common-project/hadoop-common/src/config.h.cmake new file mode 100644 index 0000000000..9098b68b87 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/config.h.cmake @@ -0,0 +1,10 @@ +#ifndef CONFIG_H +#define CONFIG_H + +#cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@" +#cmakedefine HADOOP_RUNAS_HOME "@HADOOP_RUNAS_HOME@" +#cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@" +#cmakedefine HAVE_SYNC_FILE_RANGE +#cmakedefine HAVE_POSIX_FADVISE + +#endif diff --git a/hadoop-common-project/hadoop-common/src/main/native/.autom4te.cfg b/hadoop-common-project/hadoop-common/src/main/native/.autom4te.cfg deleted file mode 100644 index a69c197883..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/native/.autom4te.cfg +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# autom4te configuration for hadoop-native library -# - -begin-language: "Autoheader-preselections" -args: --no-cache -end-language: "Autoheader-preselections" - -begin-language: "Automake-preselections" -args: --no-cache -end-language: "Automake-preselections" - -begin-language: "Autoreconf-preselections" -args: --no-cache -end-language: "Autoreconf-preselections" - -begin-language: "Autoconf-without-aclocal-m4" -args: --no-cache -end-language: "Autoconf-without-aclocal-m4" - -begin-language: "Autoconf" -args: --no-cache -end-language: "Autoconf" - diff --git a/hadoop-common-project/hadoop-common/src/main/native/Makefile.am b/hadoop-common-project/hadoop-common/src/main/native/Makefile.am deleted file mode 100644 index c4ca564c2b..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/native/Makefile.am +++ /dev/null @@ -1,66 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Notes: -# 1. This makefile is designed to do the actual builds in $(HADOOP_PREFIX)/build/native/${os.name}-${os-arch}. -# 2. This makefile depends on the following environment variables to function correctly: -# * HADOOP_NATIVE_SRCDIR -# * JAVA_HOME -# * JVM_DATA_MODEL -# * OS_NAME -# * OS_ARCH -# All these are setup by build.xml. -# - -# Export $(PLATFORM) to prevent proliferation of sub-shells -export PLATFORM = $(shell echo $$OS_NAME | tr [A-Z] [a-z]) - -ACLOCAL_AMFLAGS = -I m4 -AM_CPPFLAGS = @JNI_CPPFLAGS@ -I$(HADOOP_NATIVE_SRCDIR)/src \ - -I$(HADOOP_NATIVE_SRCDIR)/javah -AM_LDFLAGS = @JNI_LDFLAGS@ -AM_CFLAGS = -g -Wall -fPIC -O2 -if SPECIFY_DATA_MODEL -AM_LDFLAGS += -m$(JVM_DATA_MODEL) -AM_CFLAGS += -m$(JVM_DATA_MODEL) -endif - -lib_LTLIBRARIES = libhadoop.la -libhadoop_la_SOURCES = src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c \ - src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c \ - src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c \ - src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c \ - src/org/apache/hadoop/io/compress/lz4/lz4.c \ - src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c \ - src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c \ - src/org/apache/hadoop/security/getGroup.c \ - src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c \ - src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c \ - src/org/apache/hadoop/io/nativeio/file_descriptor.c \ - src/org/apache/hadoop/io/nativeio/errno_enum.c \ - src/org/apache/hadoop/io/nativeio/NativeIO.c \ - src/org/apache/hadoop/util/NativeCrc32.c \ - src/org/apache/hadoop/util/bulk_crc32.c - -libhadoop_la_LDFLAGS = -version-info 1:0:0 $(AM_LDFLAGS) -libhadoop_la_LIBADD = -ldl -ljvm - -# -#vim: sw=4: ts=4: noet -# diff --git a/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 b/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 deleted file mode 100644 index 93e05b8148..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 +++ /dev/null @@ -1,28 +0,0 @@ -# AC_COMPUTE_NEEDED_DSO(LIBRARY, TEST_PROGRAM, PREPROC_SYMBOL) -# -------------------------------------------------- -# Compute the 'actual' dynamic-library used -# for LIBRARY and set it to PREPROC_SYMBOL -AC_DEFUN([AC_COMPUTE_NEEDED_DSO], -[ -AC_CACHE_CHECK([Checking for the 'actual' dynamic-library for '-l$1'], ac_cv_libname_$1, - [ - echo '$2' > conftest.c - if test -z "`${CC} ${LDFLAGS} -o conftest conftest.c -l$1 2>&1`"; then - dnl Try objdump and ldd in that order to get the dynamic library - if test ! -z "`which objdump | grep -v 'no objdump'`"; then - ac_cv_libname_$1="`objdump -p conftest | grep NEEDED | grep $1 | sed 's/\W*NEEDED\W*\(.*\)\W*$/\"\1\"/'`" - elif test ! -z "`which ldd | grep -v 'no ldd'`"; then - ac_cv_libname_$1="`ldd conftest | grep $1 | sed 's/^[[[^A-Za-z0-9]]]*\([[[A-Za-z0-9\.]]]*\)[[[^A-Za-z0-9]]]*=>.*$/\"\1\"/'`" - elif test ! -z "`which otool | grep -v 'no otool'`"; then - ac_cv_libname_$1=\"`otool -L conftest | grep $1 | sed -e 's/^[ ]*//' -e 's/ .*//' -e 's/.*\/\(.*\)$/\1/'`\"; - else - AC_MSG_ERROR(Can't find either 'objdump' or 'ldd' or 'otool' to compute the dynamic library for '-l$1') - fi - else - ac_cv_libname_$1=libnotfound.so - fi - rm -f conftest* - ] -) -AC_DEFINE_UNQUOTED($3, ${ac_cv_libname_$1}, [The 'actual' dynamic-library for '-l$1']) -])# AC_COMPUTE_NEEDED_DSO diff --git a/hadoop-common-project/hadoop-common/src/main/native/configure.ac b/hadoop-common-project/hadoop-common/src/main/native/configure.ac deleted file mode 100644 index 34408d6418..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/native/configure.ac +++ /dev/null @@ -1,130 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# configure.ac for hadoop native code. -# - -# Notes: -# 1. This configure.ac depends on the following environment variables to function correctly: -# * HADOOP_NATIVE_SRCDIR -# * JAVA_HOME -# * JVM_DATA_MODEL -# * OS_NAME -# * OS_ARCH -# All these are setup by build.xml. - -# -*- Autoconf -*- -# Process this file with autoconf to produce a configure script. -# - -AC_PREREQ(2.59) -AC_INIT(src/org_apache_hadoop.h) -AC_CONFIG_SRCDIR([src/org_apache_hadoop.h]) -AC_CONFIG_AUX_DIR([config]) -AC_CONFIG_MACRO_DIR([m4]) -AC_CONFIG_HEADER([config.h]) -AC_SYS_LARGEFILE -AC_GNU_SOURCE - -AM_INIT_AUTOMAKE(hadoop,1.0.0) - -# Checks for programs. -AC_PROG_CC -AC_PROG_LIBTOOL - -# Checks for libraries. -dnl Check for '-ldl' -AC_CHECK_LIB([dl], [dlopen]) - -dnl Check for '-ljvm' -JNI_LDFLAGS="" -if test $JAVA_HOME != "" -then - JNI_LDFLAGS="-L$JAVA_HOME/jre/lib/$OS_ARCH/server" - JVMSOPATH=`find $JAVA_HOME/jre/ -name libjvm.so | head -n 1` - JNI_LDFLAGS="$JNI_LDFLAGS -L`dirname $JVMSOPATH`" -fi -LDFLAGS="$LDFLAGS $JNI_LDFLAGS" -AC_CHECK_LIB([jvm], [JNI_GetCreatedJavaVMs]) -AC_SUBST([JNI_LDFLAGS]) - -# Checks for header files. -dnl Check for Ansi C headers -AC_HEADER_STDC - -dnl Check for other standard C headers -AC_CHECK_HEADERS([stdio.h stddef.h], [], AC_MSG_ERROR(Some system headers not found... please ensure their presence on your platform.)) - -dnl Check for JNI headers -JNI_CPPFLAGS="" -if test $JAVA_HOME != "" -then - for dir in `find $JAVA_HOME/include -follow -type d` - do - JNI_CPPFLAGS="$JNI_CPPFLAGS -I$dir" - done -fi -cppflags_bak=$CPPFLAGS -CPPFLAGS="$CPPFLAGS $JNI_CPPFLAGS" -AC_CHECK_HEADERS([jni.h], [], AC_MSG_ERROR([Native java headers not found. Is \$JAVA_HOME set correctly?])) -CPPFLAGS=$cppflags_bak -AC_SUBST([JNI_CPPFLAGS]) - -dnl Check for zlib headers -AC_CHECK_HEADERS([zlib.h zconf.h], - AC_COMPUTE_NEEDED_DSO(z, - [#include "zlib.h" - int main(int argc, char **argv){zlibVersion();return 0;}], - HADOOP_ZLIB_LIBRARY), - AC_MSG_ERROR(Zlib headers were not found... native-hadoop library needs zlib to build. Please install the requisite zlib development package.)) - -dnl Check for snappy headers -AC_CHECK_HEADERS([snappy-c.h], - AC_COMPUTE_NEEDED_DSO(snappy, - [#include "snappy-c.h" - int main(int argc, char **argv){snappy_compress(0,0,0,0);return 0;}], - HADOOP_SNAPPY_LIBRARY), - AC_MSG_WARN(Snappy headers were not found... building without snappy.)) - -dnl Check for headers needed by the native Group resolution implementation -AC_CHECK_HEADERS([fcntl.h stdlib.h string.h unistd.h], [], AC_MSG_ERROR(Some system headers not found... please ensure their presence on your platform.)) - -dnl check for posix_fadvise -AC_CHECK_HEADERS(fcntl.h, [AC_CHECK_FUNCS(posix_fadvise)]) - -dnl check for sync_file_range -AC_CHECK_HEADERS(fcntl.h, [AC_CHECK_FUNCS(sync_file_range)]) - -# Checks for typedefs, structures, and compiler characteristics. -AC_C_CONST - -# Checks for library functions. -AC_CHECK_FUNCS([memset]) - -# Check for nonstandard STRERROR_R -AC_FUNC_STRERROR_R - -AM_CONDITIONAL([SPECIFY_DATA_MODEL], [case $host_cpu in arm*) false;; *) true;; esac]) - -AC_CONFIG_FILES([Makefile]) -AC_OUTPUT - -# -#vim: sw=2: ts=2: noet -# diff --git a/hadoop-common-project/hadoop-common/src/main/native/lib/Makefile.am b/hadoop-common-project/hadoop-common/src/main/native/lib/Makefile.am deleted file mode 100644 index 9b536ff440..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/native/lib/Makefile.am +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Makefile template for building libhadoop.so -# - -# -# Notes: -# 1. This makefile is designed to do the actual builds in $(HADOOP_PREFIX)/build/native/${os.name}-${os.arch}/lib -# 2. This makefile depends on the following environment variables to function correctly: -# * HADOOP_NATIVE_SRCDIR -# * JAVA_HOME -# * OS_ARCH -# All these are setup by build.xml and/or the top-level makefile. -# - -# Add .lo files in $(SUBDIRS) to construct libhadoop.so -HADOOP_OBJS = $(foreach path,$(addprefix ../,$(SUBDIRS)),$(wildcard $(path)/*.lo)) -AM_LDFLAGS = @JNI_LDFLAGS@ -if SPECIFY_DATA_MODEL -AM_LDFLAGS += -m$(JVM_DATA_MODEL) -endif - -lib_LTLIBRARIES = libhadoop.la -libhadoop_la_SOURCES = -libhadoop_la_LDFLAGS = -version-info 1:0:0 $(AM_LDFLAGS) -libhadoop_la_LIBADD = $(HADOOP_OBJS) -ldl -ljvm - -# -#vim: sw=4: ts=4: noet -# diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c index d52a4f6b2a..641ecd73b7 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c @@ -16,10 +16,7 @@ * limitations under the License. */ -#if defined HAVE_CONFIG_H - #include -#endif - +#include "config.h" #include "org_apache_hadoop.h" #include "org_apache_hadoop_io_compress_lz4_Lz4Compressor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c index 547b027cc1..3eebc1859d 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c @@ -16,10 +16,7 @@ * limitations under the License. */ -#if defined HAVE_CONFIG_H - #include -#endif - +#include "config.h" #include "org_apache_hadoop.h" #include "org_apache_hadoop_io_compress_lz4_Lz4Decompressor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c index 13991c23f4..96a2402ae7 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c @@ -16,36 +16,12 @@ * limitations under the License. */ -#if defined HAVE_CONFIG_H - #include -#endif - -#if defined HADOOP_SNAPPY_LIBRARY - -#if defined HAVE_STDIO_H - #include -#else - #error 'stdio.h not found' -#endif - -#if defined HAVE_STDLIB_H - #include -#else - #error 'stdlib.h not found' -#endif - -#if defined HAVE_STRING_H - #include -#else - #error 'string.h not found' -#endif - -#if defined HAVE_DLFCN_H - #include -#else - #error 'dlfcn.h not found' -#endif +#include +#include +#include +#include +#include "config.h" #include "org_apache_hadoop_io_compress_snappy.h" #include "org_apache_hadoop_io_compress_snappy_SnappyCompressor.h" @@ -123,5 +99,3 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso return (jint)compressed_direct_buf_len; } - -#endif //define HADOOP_SNAPPY_LIBRARY diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c index 767c5f4b31..a5f07ca556 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c @@ -16,36 +16,12 @@ * limitations under the License. */ -#if defined HAVE_CONFIG_H - #include -#endif - -#if defined HADOOP_SNAPPY_LIBRARY - -#if defined HAVE_STDIO_H - #include -#else - #error 'stdio.h not found' -#endif - -#if defined HAVE_STDLIB_H - #include -#else - #error 'stdlib.h not found' -#endif - -#if defined HAVE_STRING_H - #include -#else - #error 'string.h not found' -#endif - -#if defined HAVE_DLFCN_H - #include -#else - #error 'dlfcn.h not found' -#endif +#include +#include +#include +#include +#include "config.h" #include "org_apache_hadoop_io_compress_snappy.h" #include "org_apache_hadoop_io_compress_snappy_SnappyDecompressor.h" @@ -127,5 +103,3 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyDecompres return (jint)uncompressed_direct_buf_len; } - -#endif //define HADOOP_SNAPPY_LIBRARY diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h index 815e030673..3e99d5d20d 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h @@ -17,42 +17,13 @@ */ -#if !defined ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H +#ifndef ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H #define ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H - -#if defined HAVE_CONFIG_H - #include -#endif - -#if defined HADOOP_SNAPPY_LIBRARY - - #if defined HAVE_STDDEF_H - #include - #else - #error 'stddef.h not found' - #endif - - #if defined HAVE_SNAPPY_C_H - #include - #else - #error 'Please install snappy-development packages for your platform.' - #endif - - #if defined HAVE_DLFCN_H - #include - #else - #error "dlfcn.h not found" - #endif - - #if defined HAVE_JNI_H - #include - #else - #error 'jni.h not found' - #endif - - #include "org_apache_hadoop.h" - -#endif //define HADOOP_SNAPPY_LIBRARY +#include "org_apache_hadoop.h" +#include +#include +#include +#include #endif //ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am deleted file mode 100644 index 821f33f052..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am +++ /dev/null @@ -1,53 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Makefile template for building native 'zlib' for hadoop. -# - -# -# Notes: -# 1. This makefile is designed to do the actual builds in $(HADOOP_PREFIX)/build/native/${os.name}-${os.arch}/$(subdir) . -# 2. This makefile depends on the following environment variables to function correctly: -# * HADOOP_NATIVE_SRCDIR -# * JAVA_HOME -# * JVM_DATA_MODEL -# * OS_ARCH -# * PLATFORM -# All these are setup by build.xml and/or the top-level makefile. -# 3. The creation of requisite jni headers/stubs are also done by build.xml and they are -# assumed to be in $(HADOOP_PREFIX)/build/native/src/org/apache/hadoop/io/compress/zlib. -# - -# The 'vpath directive' to locate the actual source files -vpath %.c $(HADOOP_NATIVE_SRCDIR)/$(subdir) - -AM_CPPFLAGS = @JNI_CPPFLAGS@ -I$(HADOOP_NATIVE_SRCDIR)/src -AM_LDFLAGS = @JNI_LDFLAGS@ -AM_CFLAGS = -g -Wall -fPIC -O2 -if SPECIFY_DATA_MODEL -AM_CFLAGS += -m$(JVM_DATA_MODEL) -endif - -noinst_LTLIBRARIES = libnativezlib.la -libnativezlib_la_SOURCES = ZlibCompressor.c ZlibDecompressor.c -libnativezlib_la_LIBADD = -ldl -ljvm - -# -#vim: sw=4: ts=4: noet -# diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c index 9ada3f03b0..689c783ef7 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c @@ -16,34 +16,12 @@ * limitations under the License. */ -#if defined HAVE_CONFIG_H - #include -#endif - -#if defined HAVE_STDIO_H - #include -#else - #error 'stdio.h not found' -#endif - -#if defined HAVE_STDLIB_H - #include -#else - #error 'stdlib.h not found' -#endif - -#if defined HAVE_STRING_H - #include -#else - #error 'string.h not found' -#endif - -#if defined HAVE_DLFCN_H - #include -#else - #error 'dlfcn.h not found' -#endif +#include +#include +#include +#include +#include "config.h" #include "org_apache_hadoop_io_compress_zlib.h" #include "org_apache_hadoop_io_compress_zlib_ZlibCompressor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c index 3047dba267..6abe36381f 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c @@ -16,34 +16,12 @@ * limitations under the License. */ -#if defined HAVE_CONFIG_H - #include -#endif - -#if defined HAVE_STDIO_H - #include -#else - #error 'stdio.h not found' -#endif - -#if defined HAVE_STDLIB_H - #include -#else - #error 'stdlib.h not found' -#endif - -#if defined HAVE_STRING_H - #include -#else - #error 'string.h not found' -#endif - -#if defined HAVE_DLFCN_H - #include -#else - #error 'dlfcn.h not found' -#endif +#include +#include +#include +#include +#include "config.h" #include "org_apache_hadoop_io_compress_zlib.h" #include "org_apache_hadoop_io_compress_zlib_ZlibDecompressor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h index 16b607b4a9..c53aa531c9 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h @@ -19,40 +19,13 @@ #if !defined ORG_APACHE_HADOOP_IO_COMPRESS_ZLIB_ZLIB_H #define ORG_APACHE_HADOOP_IO_COMPRESS_ZLIB_ZLIB_H -#if defined HAVE_CONFIG_H - #include -#endif - -#if defined HAVE_STDDEF_H - #include -#else - #error 'stddef.h not found' -#endif - -#if defined HAVE_ZLIB_H - #include -#else - #error 'Please install zlib-development packages for your platform.' -#endif - -#if defined HAVE_ZCONF_H - #include -#else - #error 'Please install zlib-development packages for your platform.' -#endif - -#if defined HAVE_DLFCN_H - #include -#else - #error "dlfcn.h not found" -#endif - -#if defined HAVE_JNI_H - #include -#else - #error 'jni.h not found' -#endif +#include +#include +#include +#include +#include +#include "config.h" #include "org_apache_hadoop.h" /* A helper macro to convert the java 'stream-handle' to a z_stream pointer. */ diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index fbcf9563ee..c08ea037d9 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -16,9 +16,6 @@ * limitations under the License. */ -// get the autoconf settings -#include "config.h" - #include #include #include @@ -32,6 +29,7 @@ #include #include +#include "config.h" #include "org_apache_hadoop.h" #include "org_apache_hadoop_io_nativeio_NativeIO.h" #include "file_descriptor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c index 869c2ba2e8..dd51c0a257 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c @@ -16,9 +16,6 @@ * limitations under the License. */ -// get the autoconf settings -#include "config.h" - #include #include #include @@ -26,6 +23,7 @@ #include #include +#include "config.h" #include "org_apache_hadoop.h" #include "org_apache_hadoop_util_NativeCrc32.h" #include "gcc_optimizations.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h b/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h index 7a777c2f4f..a50c41dbbb 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h @@ -24,21 +24,10 @@ #if !defined ORG_APACHE_HADOOP_H #define ORG_APACHE_HADOOP_H -#if defined HAVE_CONFIG_H - #include -#endif - -#if defined HAVE_DLFCN_H - #include -#else - #error "dlfcn.h not found" -#endif +#include +#include -#if defined HAVE_JNI_H - #include -#else - #error 'jni.h not found' -#endif +#include "config.h" /* A helper macro to 'throw' a java exception. */ #define THROW(env, exception_name, message) \ diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index ebff0f57f5..c775c51e3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -415,76 +415,22 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> maven-antrun-plugin - compile + make compile - - run - + run - - - - + + + + + + + - - - - org.codehaus.mojo - make-maven-plugin - - - compile - compile - - autoreconf - configure - make-install - - - - ${project.build.directory}/native - - -i - -f - - - - - - ac_cv_func_malloc_0_nonnull - yes - - - JVM_ARCH - ${sun.arch.data.model} - - - - - ${project.build.directory}/native - /usr/local - - - - - ac_cv_func_malloc_0_nonnull - yes - - - JVM_ARCH - ${sun.arch.data.model} - - - - - ${project.build.directory}/native/target - - - - - - 4.0.0 - - org.apache.hadoop - hadoop-project - 3.0.0-SNAPSHOT - ../../../../../hadoop-project - - org.apache.hadoop.contrib - hadoop-hdfs-fuse - 3.0.0-SNAPSHOT - pom - - Apache Hadoop HDFS Fuse - Apache Hadoop HDFS Fuse - - - - org.apache.hadoop - hadoop-hdfs - compile - - - org.apache.hadoop - hadoop-hdfs - test - test-jar - - - - - - - - - org.apache.maven.plugins - maven-eclipse-plugin - 2.6 - - - org.apache.maven.plugins - maven-surefire-plugin - - 1 - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - - javadoc - - site - - true - true - false - ${maven.compile.source} - ${maven.compile.encoding} - - - HttpFs API - * - - - - - - - - org.apache.maven.plugins - maven-project-info-reports-plugin - - - - false - - - dependencies - - site - - - - - org.apache.rat - apache-rat-plugin - - - - - - - - - - - fuse - - false - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - prepare-compile-native - generate-sources - - run - - - - - - - - - - - compile-fuse - compile - - run - - - - - - - - - - - - - - - - diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/CMakeLists.txt new file mode 100644 index 0000000000..fb3c580e94 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/CMakeLists.txt @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Find Linux FUSE +IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + find_package(PkgConfig REQUIRED) + pkg_check_modules(FUSE fuse) + IF(FUSE_FOUND) + FLATTEN_LIST("${FUSE_CFLAGS}" " " FUSE_CFLAGS) + FLATTEN_LIST("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FUSE_CFLAGS}") + set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} ${FUSE_LDFLAGS}") + MESSAGE(STATUS "Building Linux FUSE client.") + include_directories(${FUSE_INCLUDE_DIRS}) + ELSE(FUSE_FOUND) + MESSAGE(STATUS "Failed to find Linux FUSE libraries or include files. Will not build FUSE client.") + ENDIF(FUSE_FOUND) +ELSE (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + MESSAGE(STATUS "Non-Linux system detected. Will not build FUSE client.") +ENDIF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + +IF(FUSE_FOUND) + add_executable(fuse_dfs + fuse_dfs.c + fuse_options.c + fuse_connect.c + fuse_impls_access.c + fuse_impls_chmod.c + fuse_impls_chown.c + fuse_impls_create.c + fuse_impls_flush.c + fuse_impls_getattr.c + fuse_impls_mkdir.c + fuse_impls_mknod.c + fuse_impls_open.c + fuse_impls_read.c + fuse_impls_readdir.c + fuse_impls_release.c + fuse_impls_rename.c + fuse_impls_rmdir.c + fuse_impls_statfs.c + fuse_impls_symlink.c + fuse_impls_truncate.c + fuse_impls_unlink.c + fuse_impls_utimens.c + fuse_impls_write.c + fuse_init.c + fuse_stat_struct.c + fuse_trash.c + fuse_users.c + ) + target_link_libraries(fuse_dfs + ${FUSE_LIBRARIES} + ${JAVA_JVM_LIBRARY} + hdfs + m + ) +ENDIF(FUSE_FOUND) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/Makefile.am b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/Makefile.am deleted file mode 100644 index 706297f314..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/Makefile.am +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -bin_PROGRAMS = fuse_dfs -fuse_dfs_SOURCES = fuse_dfs.c fuse_options.c fuse_trash.c fuse_stat_struct.c fuse_users.c fuse_init.c fuse_connect.c fuse_impls_access.c fuse_impls_chmod.c fuse_impls_chown.c fuse_impls_create.c fuse_impls_flush.c fuse_impls_getattr.c fuse_impls_mkdir.c fuse_impls_mknod.c fuse_impls_open.c fuse_impls_read.c fuse_impls_release.c fuse_impls_readdir.c fuse_impls_rename.c fuse_impls_rmdir.c fuse_impls_statfs.c fuse_impls_symlink.c fuse_impls_truncate.c fuse_impls_utimens.c fuse_impls_unlink.c fuse_impls_write.c -AM_CFLAGS= -Wall -g -AM_CPPFLAGS= -DPERMS=$(PERMS) -D_FILE_OFFSET_BITS=64 -I$(JAVA_HOME)/include -I$(HADOOP_PREFIX)/../../src/main/native -I$(JAVA_HOME)/include/linux -D_FUSE_DFS_VERSION=\"$(PACKAGE_VERSION)\" -DPROTECTED_PATHS=\"$(PROTECTED_PATHS)\" -I$(FUSE_HOME)/include -AM_LDFLAGS= -L$(HADOOP_PREFIX)/../../target/native/target/usr/local/lib64 -L$(HADOOP_PREFIX)/../../target/native/target/usr/local/lib -L$(FUSE_HOME)/lib -L$(JAVA_HOME)/jre/lib/$(OS_ARCH)/server -fuse_dfs_LDADD=-lfuse -lhdfs -ljvm -lm diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_dfs.h b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_dfs.h index 56ed9cb173..4554dbdbea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_dfs.h +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_dfs.h @@ -31,13 +31,9 @@ #include #include -#ifdef HAVE_CONFIG_H -#include -#endif - -#ifdef HAVE_SETXATTR #include -#endif + +#include "config.h" // // Check if a path is in the mount option supplied protected paths. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/Makefile.am b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/Makefile.am deleted file mode 100644 index 8bbd627315..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/Makefile.am +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -@PRODUCT_MK@ - -#AM_CPPFLAGS = -I$(top_srcdir) -ACLOCAL_AMFLAGS = -I m4 - -lib_LTLIBRARIES = libhdfs.la -libhdfs_la_SOURCES = hdfs.c hdfsJniHelper.c hdfs.h - -#check_PROGRAMS = hdfs_test hdfs_read hdfs_write -check_PROGRAMS = hdfs_test hdfs_read hdfs_write - -hdfs_test_SOURCES = hdfs_test.c hdfs.h -hdfs_test_LDADD = ${libdir}/libhdfs.la - -hdfs_read_SOURCES = hdfs_read.c -hdfs_read_LDADD = ${libdir}/libhdfs.la - -hdfs_write_SOURCES = hdfs_write.c -hdfs_write_LDADD = ${libdir}/libhdfs.la - -test: hdfs_test hdfs_read hdfs_write - ${LIBHDFS_SRC_DIR}/tests/test-libhdfs.sh - - -# vim: sw=4: ts=4: noet diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/configure.ac b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/configure.ac deleted file mode 100644 index d801fc4738..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/configure.ac +++ /dev/null @@ -1,125 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Autoconf input file -# $Id$ - -AC_INIT([libhdfs], [0.1.0], omalley@apache.org) -AC_PREFIX_DEFAULT([`pwd`/../install]) -AC_CONFIG_AUX_DIR([config]) - -# Generates Makefile from Makefile.am. Modify when new subdirs are added. -# Change Makefile.am also to add subdirectly. -AM_INIT_AUTOMAKE(foreign no-dist) -AC_CONFIG_FILES(Makefile) - -LT_INIT - -AC_CONFIG_MACRO_DIR([m4]) -dnl ------------------------------------------------------------------------- -dnl Check current host (forget about cross compilation) and validate it -dnl against the cache (fail if the cache differs) -dnl ------------------------------------------------------------------------- -AP_MSG_HEADER([Current host]) -AC_CANONICAL_HOST() -AP_CANONICAL_HOST_CHECK() - -dnl ------------------------------------------------------------------------- -dnl Check C environment -dnl ------------------------------------------------------------------------- -AP_MSG_HEADER([C-Language compilation tools]) -AC_PROG_CC() -AC_CHECK_TOOL(RANLIB, ranlib, :) - -dnl ------------------------------------------------------------------------- -dnl Check if this host is supported -dnl ------------------------------------------------------------------------- -AP_MSG_HEADER([Host support]) -AP_SUPPORTED_HOST() -if test "$supported_os" = "darwin" -then - if test -z "$JAVA_HOME" -a -d /System/Library/Frameworks/JavaVM.framework/Home; then - JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Home - fi - - _prevdir=`/bin/pwd` - if test -n "$JAVA_HOME" -a -d "$JAVA_HOME/include"; then - cd "$JAVA_HOME/include" - elif test -n "$JAVA_HOME" -a -d "$JAVA_HOME/../Headers"; then - cd "$JAVA_HOME/../Headers" - else - cd /System/Library/Frameworks/JavaVM.framework/Headers - fi - CFLAGS="$CFLAGS -m${JVM_ARCH} -I`/bin/pwd -P`" - cd $_prevdir - unset _prevdir -fi - -dnl ------------------------------------------------------------------------- -dnl Check JAVA environment -dnl ------------------------------------------------------------------------- -AP_MSG_HEADER([Java compilation tools]) -AP_JAVA() -AP_SABLEVM() -AP_KAFFE() -AP_PROG_JAVAC() -AP_PROG_JAR() -AP_JVM_LIBDIR() -if test "$supported_os" != "darwin" -then - case $host_cpu in - arm*) ;; - *) - CFLAGS="$CFLAGS -m${JVM_ARCH}" - LDFLAGS="$LDFLAGS -m${JVM_ARCH}" - ;; - esac - AC_MSG_RESULT([VALUE OF JVM_ARCH IS :$JVM_ARCH]) - CFLAGS="$CFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/$supported_os" - LDFLAGS="$LDFLAGS -L$LIB_JVM_DIR -ljvm -Wl,-x" -fi - -dnl ------------------------------------------------------------------------- -dnl Add gcc specific CFLAGS. -dnl ------------------------------------------------------------------------- -if test "$GCC" = "yes" -then - CFLAGS="$CFLAGS -Wall -Wstrict-prototypes" - AC_MSG_RESULT([gcc flags added]) -fi -dnl ------------------------------------------------------------------------- -dnl Add gcc specific CFLAGS. -dnl ------------------------------------------------------------------------- -if test -z "$LDCMD" -then - LDCMD="$CC" -fi -AC_SUBST(LDCMD) - - -AC_PROG_CC -AC_PROG_LIBTOOL - -AC_TYPE_SIZE_T -AC_CHECK_FUNCS([strdup strerror strtoul]) -AC_CHECK_HEADERS([fcntl.h]) -AC_C_CONST -AC_C_VOLATILE -#AC_FUNC_MALLOC -AC_HEADER_STDBOOL -AC_SUBST(PRODUCT_MK) -AC_OUTPUT diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apfunctions.m4 b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apfunctions.m4 deleted file mode 100644 index cb5938ffca..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apfunctions.m4 +++ /dev/null @@ -1,41 +0,0 @@ -dnl -dnl Licensed to the Apache Software Foundation (ASF) under one or more -dnl contributor license agreements. See the NOTICE file distributed with -dnl this work for additional information regarding copyright ownership. -dnl The ASF licenses this file to You under the Apache License, Version 2.0 -dnl (the "License"); you may not use this file except in compliance with -dnl the License. You may obtain a copy of the License at -dnl -dnl http://www.apache.org/licenses/LICENSE-2.0 -dnl -dnl Unless required by applicable law or agreed to in writing, software -dnl distributed under the License is distributed on an "AS IS" BASIS, -dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -dnl See the License for the specific language governing permissions and -dnl limitations under the License. -dnl - -dnl ------------------------------------------------------------------------- -dnl Author Pier Fumagalli -dnl Version $Id$ -dnl ------------------------------------------------------------------------- - -AC_DEFUN([AP_MSG_HEADER],[ - printf "*** %s ***\n" "$1" 1>&2 - AC_PROVIDE([$0]) -]) - -AC_DEFUN([AP_CANONICAL_HOST_CHECK],[ - AC_MSG_CHECKING([cached host system type]) - if { test x"${ac_cv_host_system_type+set}" = x"set" && - test x"$ac_cv_host_system_type" != x"$host" ; } - then - AC_MSG_RESULT([$ac_cv_host_system_type]) - AC_MSG_ERROR([remove the \"$cache_file\" file and re-run configure]) - else - AC_MSG_RESULT(ok) - ac_cv_host_system_type="$host" - fi - AC_PROVIDE([$0]) -]) - diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apjava.m4 b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apjava.m4 deleted file mode 100644 index 993fc5bed9..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apjava.m4 +++ /dev/null @@ -1,142 +0,0 @@ -dnl -dnl Licensed to the Apache Software Foundation (ASF) under one or more -dnl contributor license agreements. See the NOTICE file distributed with -dnl this work for additional information regarding copyright ownership. -dnl The ASF licenses this file to You under the Apache License, Version 2.0 -dnl (the "License"); you may not use this file except in compliance with -dnl the License. You may obtain a copy of the License at -dnl -dnl http://www.apache.org/licenses/LICENSE-2.0 -dnl -dnl Unless required by applicable law or agreed to in writing, software -dnl distributed under the License is distributed on an "AS IS" BASIS, -dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -dnl See the License for the specific language governing permissions and -dnl limitations under the License. -dnl - -dnl ------------------------------------------------------------------------- -dnl Author Pier Fumagalli -dnl Version $Id$ -dnl ------------------------------------------------------------------------- - -AC_DEFUN([AP_PROG_JAVAC_WORKS],[ - AC_CACHE_CHECK([wether the Java compiler ($JAVAC) works],ap_cv_prog_javac_works,[ - echo "public class Test {}" > Test.java - $JAVAC $JAVACFLAGS Test.java > /dev/null 2>&1 - if test $? -eq 0 - then - rm -f Test.java Test.class - ap_cv_prog_javac_works=yes - else - rm -f Test.java Test.class - AC_MSG_RESULT(no) - AC_MSG_ERROR([installation or configuration problem: javac cannot compile]) - fi - ]) -]) - -dnl Check for JAVA compilers. -AC_DEFUN([AP_PROG_JAVAC],[ - if test "$SABLEVM" != "NONE" - then - AC_PATH_PROG(JAVACSABLE,javac-sablevm,NONE,$JAVA_HOME/bin) - else - JAVACSABLE="NONE" - fi - if test "$JAVACSABLE" = "NONE" - then - XPATH="$JAVA_HOME/bin:$JAVA_HOME/Commands:$PATH" - AC_PATH_PROG(JAVAC,javac,NONE,$XPATH) - else - AC_PATH_PROG(JAVAC,javac-sablevm,NONE,$JAVA_HOME/bin) - fi - AC_MSG_RESULT([$JAVAC]) - if test "$JAVAC" = "NONE" - then - AC_MSG_ERROR([javac not found]) - fi - AP_PROG_JAVAC_WORKS() - AC_PROVIDE([$0]) - AC_SUBST(JAVAC) - AC_SUBST(JAVACFLAGS) -]) - -dnl Check for jar archivers. -AC_DEFUN([AP_PROG_JAR],[ - if test "$SABLEVM" != "NONE" - then - AC_PATH_PROG(JARSABLE,jar-sablevm,NONE,$JAVA_HOME/bin) - else - JARSABLE="NONE" - fi - if test "$JARSABLE" = "NONE" - then - XPATH="$JAVA_HOME/bin:$JAVA_HOME/Commands:$PATH" - AC_PATH_PROG(JAR,jar,NONE,$XPATH) - else - AC_PATH_PROG(JAR,jar-sablevm,NONE,$JAVA_HOME/bin) - fi - if test "$JAR" = "NONE" - then - AC_MSG_ERROR([jar not found]) - fi - AC_PROVIDE([$0]) - AC_SUBST(JAR) -]) - -AC_DEFUN([AP_JAVA],[ - AC_ARG_WITH(java,[ --with-java=DIR Specify the location of your JDK installation],[ - AC_MSG_CHECKING([JAVA_HOME]) - if test -d "$withval" - then - JAVA_HOME="$withval" - AC_MSG_RESULT([$JAVA_HOME]) - else - AC_MSG_RESULT([failed]) - AC_MSG_ERROR([$withval is not a directory]) - fi - AC_SUBST(JAVA_HOME) - ]) - if test x"$JAVA_HOME" = x - then - AC_MSG_ERROR([Java Home not defined. Rerun with --with-java=[...] parameter]) - fi -]) - -dnl check if the JVM in JAVA_HOME is sableVM -dnl $JAVA_HOME/bin/sablevm and /opt/java/lib/sablevm/bin are tested. -AC_DEFUN([AP_SABLEVM],[ - if test x"$JAVA_HOME" != x - then - AC_PATH_PROG(SABLEVM,sablevm,NONE,$JAVA_HOME/bin) - if test "$SABLEVM" = "NONE" - then - dnl java may be SableVM. - if $JAVA_HOME/bin/java -version 2> /dev/null | grep SableVM > /dev/null - then - SABLEVM=$JAVA_HOME/bin/java - fi - fi - if test "$SABLEVM" != "NONE" - then - AC_MSG_RESULT([Using sableVM: $SABLEVM]) - CFLAGS="$CFLAGS -DHAVE_SABLEVM" - fi - fi -]) - -dnl check if the JVM in JAVA_HOME is kaffe -dnl $JAVA_HOME/bin/kaffe is tested. -AC_DEFUN([AP_KAFFE],[ - if test x"$JAVA_HOME" != x - then - AC_PATH_PROG(KAFFEVM,kaffe,NONE,$JAVA_HOME/bin) - if test "$KAFFEVM" != "NONE" - then - AC_MSG_RESULT([Using kaffe: $KAFFEVM]) - CFLAGS="$CFLAGS -DHAVE_KAFFEVM" - LDFLAGS="$LDFLAGS -Wl,-rpath $JAVA_HOME/jre/lib/$HOST_CPU -L $JAVA_HOME/jre/lib/$HOST_CPU -lkaffevm" - fi - fi -]) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apsupport.m4 b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apsupport.m4 deleted file mode 100644 index 0c8b262dcb..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apsupport.m4 +++ /dev/null @@ -1,168 +0,0 @@ -dnl -dnl Licensed to the Apache Software Foundation (ASF) under one or more -dnl contributor license agreements. See the NOTICE file distributed with -dnl this work for additional information regarding copyright ownership. -dnl The ASF licenses this file to You under the Apache License, Version 2.0 -dnl (the "License"); you may not use this file except in compliance with -dnl the License. You may obtain a copy of the License at -dnl -dnl http://www.apache.org/licenses/LICENSE-2.0 -dnl -dnl Unless required by applicable law or agreed to in writing, software -dnl distributed under the License is distributed on an "AS IS" BASIS, -dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -dnl See the License for the specific language governing permissions and -dnl limitations under the License. -dnl - -dnl ------------------------------------------------------------------------- -dnl Author Pier Fumagalli -dnl Version $Id$ -dnl ------------------------------------------------------------------------- - -AC_DEFUN([AP_SUPPORTED_HOST],[ - AC_MSG_CHECKING([C flags dependant on host system type]) - - case $host_os in - darwin*) - CFLAGS="$CFLAGS -DOS_DARWIN -DDSO_DYLD" - supported_os="darwin" - ;; - solaris*) - CFLAGS="$CFLAGS -DOS_SOLARIS -DDSO_DLFCN" - supported_os="solaris" - LIBS="$LIBS -ldl -lthread" - ;; - linux*) - CFLAGS="$CFLAGS -DOS_LINUX -DDSO_DLFCN" - supported_os="linux" - LIBS="$LIBS -ldl -lpthread" - ;; - cygwin) - CFLAGS="$CFLAGS -DOS_CYGWIN -DDSO_DLFCN -DNO_SETSID" - supported_os="win32" - ;; - sysv) - CFLAGS="$CFLAGS -DOS_SYSV -DDSO_DLFCN" - LIBS="$LIBS -ldl" - ;; - sysv4) - CFLAGS="$CFLAGS -DOS_SYSV -DDSO_DLFCN -Kthread" - LDFLAGS="-Kthread $LDFLAGS" - LIBS="$LIBS -ldl" - ;; - freebsd*) - CFLAGS="$CFLAGS -DOS_FREEBSD -DDSO_DLFCN -D_THREAD_SAFE -pthread" - LDFLAGS="-pthread $LDFLAGS" - supported_os="freebsd" - ;; - osf5*) - CFLAGS="$CFLAGS -pthread -DOS_TRU64 -DDSO_DLFCN -D_XOPEN_SOURCE_EXTENDED" - LDFLAGS="$LDFLAGS -pthread" - ;; - hpux11*) - CFLAGS="$CFLAGS -pthread -DOS_HPUX -DDSO_DLFCN" - LDFLAGS="$LDFLAGS -pthread" - LIBS="$LIBS -lpthread" - ;; - *) - AC_MSG_RESULT([failed]) - AC_MSG_ERROR([Unsupported operating system "$host_os"]);; - esac - - case $host_cpu in - powerpc*) - CFLAGS="$CFLAGS -DCPU=\\\"$host_cpu\\\"" - HOST_CPU=$host_cpu;; - sparc*) - CFLAGS="$CFLAGS -DCPU=\\\"$host_cpu\\\"" - HOST_CPU=$host_cpu;; - i?86) - CFLAGS="$CFLAGS -DCPU=\\\"i386\\\"" - HOST_CPU=i386;; - x86_64) - CFLAGS="$CFLAGS -DCPU=\\\"amd64\\\"" - HOST_CPU=amd64;; - bs2000) - CFLAGS="$CFLAGS -DCPU=\\\"osd\\\" -DCHARSET_EBCDIC -DOSD_POSIX" - supported_os="osd" - LDFLAGS="-Kno_link_stdlibs -B llm4" - LIBS="$LIBS -lBLSLIB" - LDCMD="/opt/C/bin/cc" - HOST_CPU=osd;; - mips) - CFLAGS="$CFLAGS -DCPU=\\\"mips\\\"" - supported_os="mips" - HOST_CPU=mips;; - alpha*) - CFLAGS="$CFLAGS -DCPU=\\\"alpha\\\"" - supported_os="alpha" - HOST_CPU=alpha;; - hppa2.0w) - CFLAGS="$CFLAGS -DCPU=\\\"PA_RISC2.0W\\\"" - supported_os="hp-ux" - HOST_CPU=PA_RISC2.0W;; - hppa2.0) - CFLAGS="$CFLAGS -DCPU=\\\"PA_RISC2.0\\\"" - supported_os="hp-ux" - HOST_CPU=PA_RISC2.0;; - mipsel) - CFLAGS="$CFLAGS -DCPU=\\\"mipsel\\\"" - supported_os="mipsel" - HOST_CPU=mipsel;; - ia64) - CFLAGS="$CFLAGS -DCPU=\\\"ia64\\\"" - supported_os="ia64" - HOST_CPU=ia64;; - s390) - CFLAGS="$CFLAGS -DCPU=\\\"s390\\\"" - supported_os="s390" - HOST_CPU=s390;; - arm*) - CFLAGS="$CFLAGS -DCPU=\\\"arm\\\"" - supported_os="arm" - HOST_CPU=arm;; - *) - AC_MSG_RESULT([failed]) - AC_MSG_ERROR([Unsupported CPU architecture "$host_cpu"]);; - esac - - AC_MSG_RESULT([ok]) - AC_SUBST(CFLAGS) - AC_SUBST(LDFLAGS) -]) - -AC_DEFUN([AP_JVM_LIBDIR],[ - AC_MSG_CHECKING([where on earth this jvm library is..]) - javabasedir=$JAVA_HOME - case $host_os in - cygwin* | mingw* | pw23* ) - lib_jvm_dir=`find $javabasedir -follow \( \ - \( -name client -type d -prune \) -o \ - \( -name "jvm.dll" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` - ;; - aix*) - lib_jvm_dir=`find $javabasedir \( \ - \( -name client -type d -prune \) -o \ - \( -name "libjvm.*" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` - if test -z "$lib_jvm_dir"; then - lib_jvm_dir=`find $javabasedir \( \ - \( -name client -type d -prune \) -o \ - \( -name "libkaffevm.*" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` - fi - ;; - *) - lib_jvm_dir=`find $javabasedir -follow \( \ - \( -name client -type d -prune \) -o \ - \( -name "libjvm.*" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` - if test -z "$lib_jvm_dir"; then - lib_jvm_dir=`find $javabasedir -follow \( \ - \( -name client -type d -prune \) -o \ - \( -name "libkaffevm.*" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` - fi - ;; - esac - LIB_JVM_DIR=$lib_jvm_dir - AC_MSG_RESULT([ohh u there ... $LIB_JVM_DIR]) - AC_SUBST(LIB_JVM_DIR) -]) diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml index 0e2684b1ea..27161004a3 100644 --- a/hadoop-hdfs-project/pom.xml +++ b/hadoop-hdfs-project/pom.xml @@ -34,7 +34,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> hadoop-hdfs hadoop-hdfs-httpfs hadoop-hdfs/src/contrib/bkjournal - hadoop-hdfs/src/contrib/fuse-dfs diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index fb3c97b521..f865b2dc59 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -47,47 +47,37 @@ - org.codehaus.mojo - make-maven-plugin + org.apache.maven.plugins + maven-antrun-plugin - compile + make compile - - autoreconf - configure - make-install - + run + + + + + + + + + + + + - test + native_tests test - - test - + + + + + + - - - ${project.build.directory}/native/container-executor - - -i - - - - - - CFLAGS - -DHADOOP_CONF_DIR=${container-executor.conf.dir} ${container-executor.additional_cflags} - - - ${project.build.directory}/native/container-executor - /usr/local - - - ${project.build.directory}/native/target - - @@ -172,14 +162,6 @@ run - - - - - - - - diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt new file mode 100644 index 0000000000..e639507d26 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 2.6 FATAL_ERROR) + +set(CMAKE_BUILD_TYPE, Release) + +if (JVM_ARCH_DATA_MODEL EQUAL 32) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") + set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32") +endif (JVM_ARCH_DATA_MODEL EQUAL 32) + +function(output_directory TGT DIR) + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") +endfunction(output_directory TGT DIR) + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2 -D_GNU_SOURCE") +# note: can't enable -D_LARGEFILE: see MAPREDUCE-4258 +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT") + +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_BINARY_DIR} + main/native/container-executor + main/native/container-executor/impl +) +CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h) + +add_library(container + main/native/container-executor/impl/configuration.c + main/native/container-executor/impl/container-executor.c +) + +add_executable(container-executor + main/native/container-executor/impl/main.c +) +target_link_libraries(container-executor + container +) +output_directory(container-executor target/usr/local/bin) + +add_executable(test-container-executor + main/native/container-executor/test/test-container-executor.c +) +target_link_libraries(test-container-executor + container +) +output_directory(test-container-executor target/usr/local/bin) diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake new file mode 100644 index 0000000000..1fff36131f --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake @@ -0,0 +1,6 @@ +#ifndef CONFIG_H +#define CONFIG_H + +#cmakedefine HADOOP_CONF_DIR "@HADOOP_CONF_DIR@" + +#endif diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.autom4te.cfg b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.autom4te.cfg deleted file mode 100644 index d21d1c9877..0000000000 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.autom4te.cfg +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# autom4te configuration for hadoop utils library -# - -begin-language: "Autoheader-preselections" -args: --no-cache -end-language: "Autoheader-preselections" - -begin-language: "Automake-preselections" -args: --no-cache -end-language: "Automake-preselections" - -begin-language: "Autoreconf-preselections" -args: --no-cache -end-language: "Autoreconf-preselections" - -begin-language: "Autoconf-without-aclocal-m4" -args: --no-cache -end-language: "Autoconf-without-aclocal-m4" - -begin-language: "Autoconf" -args: --no-cache -end-language: "Autoconf" - diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.deps/container-executor.Po b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.deps/container-executor.Po deleted file mode 100644 index 9ce06a81ea..0000000000 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.deps/container-executor.Po +++ /dev/null @@ -1 +0,0 @@ -# dummy diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/Makefile.am b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/Makefile.am deleted file mode 100644 index 4938bb2f53..0000000000 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/Makefile.am +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -AM_CFLAGS=-I$(srcdir)/impl -Wall -g -Werror - -# Define the programs that need to be built -bin_PROGRAMS = container-executor -check_PROGRAMS = test-container-executor - -TESTS = test-container-executor - -# Define the sources for the common files -common_SOURCES = impl/configuration.c impl/container-executor.c - -# Define the sources for the real executable -container_executor_SOURCES = $(common_SOURCES) impl/main.c - -# Define the sources for the test executable -test_container_executor_SOURCES = $(common_SOURCES) test/test-container-executor.c diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/configure.ac b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/configure.ac deleted file mode 100644 index db8af88cf1..0000000000 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/configure.ac +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -*- Autoconf -*- -# Process this file with autoconf to produce a configure script. - -AC_PREREQ(2.59) -AC_INIT(linux-container-executor, 1.0.0, mapreduce-dev@hadoop.apache.org) -AC_GNU_SOURCE -#AC_SYS_LARGEFILE - -AM_INIT_AUTOMAKE([subdir-objects foreign no-dist]) - -AC_CONFIG_SRCDIR([impl/container-executor.c]) -AC_CONFIG_FILES([Makefile]) - -AC_PREFIX_DEFAULT(`pwd`/../install) - -CHECK_INSTALL_CFLAG -HADOOP_UTILS_SETUP - -# Checks for programs. -AC_PROG_CC -AM_PROG_CC_C_O - -# Checks for libraries. - -# Checks for header files. -AC_LANG(C) -AC_CHECK_HEADERS([unistd.h]) - -# Checks for typedefs, structures, and compiler characteristics. -AC_HEADER_STDBOOL -AC_C_CONST -AC_TYPE_OFF_T -AC_TYPE_SIZE_T -AC_FUNC_STRERROR_R - -# Checks for library functions. -AC_CHECK_FUNCS([mkdir uname]) -AC_OUTPUT diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index cd8caabe33..d6ce5aa706 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -16,6 +16,7 @@ * limitations under the License. */ +#include "config.h" #include "configuration.h" #include "container-executor.h" @@ -29,8 +30,6 @@ #include #include -#define _STRINGIFY(X) #X -#define STRINGIFY(X) _STRINGIFY(X) #define CONF_FILENAME "container-executor.cfg" // When building as part of a Maven build this value gets defined by using @@ -101,7 +100,7 @@ int main(int argc, char **argv) { char *executable_file = get_executable(); - char *orig_conf_file = STRINGIFY(HADOOP_CONF_DIR) "/" CONF_FILENAME; + char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME; char *conf_file = resolve_config_path(orig_conf_file, argv[0]); char *local_dirs, *log_dirs; From 7073ff222e58448536e1df716a783bde95ea52fd Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Wed, 6 Jun 2012 20:56:26 +0000 Subject: [PATCH 02/91] HDFS-3505. DirectoryScanner does not join all threads in shutdown. Contributed by Colin Patrick McCabe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1347117 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/datanode/DirectoryScanner.java | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index cf0fb480f4..b2d6f86d40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -307,6 +307,9 @@ Release 2.0.1-alpha - UNRELEASED HDFS-3266. DFSTestUtil#waitCorruptReplicas doesn't sleep between checks. (Madhukara Phatak via atm) + HDFS-3505. DirectoryScanner does not join all threads in shutdown. + (Colin Patrick McCabe via eli) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java index eac68d970b..3a143357ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java @@ -302,6 +302,22 @@ void shutdown() { shouldRun = false; if (masterThread != null) masterThread.shutdown(); if (reportCompileThreadPool != null) reportCompileThreadPool.shutdown(); + if (masterThread != null) { + try { + masterThread.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + LOG.error("interrupted while waiting for masterThread to " + + "terminate", e); + } + } + if (reportCompileThreadPool != null) { + try { + reportCompileThreadPool.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + LOG.error("interrupted while waiting for reportCompileThreadPool to " + + "terminate", e); + } + } if (!retainDiffs) clear(); } From 3d543848c2d545d925c4cf39be0930606281706a Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Wed, 6 Jun 2012 21:14:15 +0000 Subject: [PATCH 03/91] HADOOP-8485. Don't hardcode "Apache Hadoop 0.23" in the docs. Contributed by Eli Collins git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1347133 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ hadoop-project/src/site/apt/index.apt.vm | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 57c387ac7d..080811afa3 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -269,6 +269,8 @@ Release 2.0.1-alpha - UNRELEASED HADOOP-8481. update BUILDING.txt to talk about cmake rather than autotools. (Colin Patrick McCabe via eli) + HADOOP-8485. Don't hardcode "Apache Hadoop 0.23" in the docs. (eli) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-project/src/site/apt/index.apt.vm b/hadoop-project/src/site/apt/index.apt.vm index fe05eebbfe..32e708ec54 100644 --- a/hadoop-project/src/site/apt/index.apt.vm +++ b/hadoop-project/src/site/apt/index.apt.vm @@ -16,10 +16,10 @@ --- ${maven.build.timestamp} -Apache Hadoop 0.23 +Apache Hadoop ${project.version} - Apache Hadoop 0.23 consists of significant improvements over the previous - stable release (hadoop-0.20.205). + Apache Hadoop ${project.version} consists of significant + improvements over the previous stable release (hadoop-1.x). Here is a short overview of the improvments to both HDFS and MapReduce. From b0508760134d18405cadbadc21a8e04b1b524b70 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 6 Jun 2012 21:19:33 +0000 Subject: [PATCH 04/91] Fixing CHANGES.txt to refer to branch-2 instead of (an unreleased, not-yet-decided) release. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1347136 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++-- hadoop-mapreduce-project/CHANGES.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 080811afa3..7c7fa5d85c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -197,7 +197,7 @@ Trunk (unreleased changes) HADOOP-8405. ZKFC tests leak ZK instances. (todd) -Release 2.0.1-alpha - UNRELEASED +Branch-2 ( Unreleased changes ) INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b2d6f86d40..24b38eb2a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -178,8 +178,8 @@ Trunk (unreleased changes) HDFS-3432. TestDFSZKFailoverController tries to fail over too early (todd) -Release 2.0.1-alpha - UNRELEASED - +Branch-2 ( Unreleased changes ) + INCOMPATIBLE CHANGES NEW FEATURES diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 56a036fc15..9641fc5ce3 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -118,7 +118,7 @@ Trunk (unreleased changes) MAPREDUCE-3990. MRBench allows Long-sized input-lines value but parses CLI argument as an Integer. (harsh) -Release 2.0.1-alpha - UNRELEASED +Branch-2 ( Unreleased changes ) INCOMPATIBLE CHANGES From 3983fdd6fc01e32f8e2b65de1fe1441a1877986c Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 6 Jun 2012 22:48:48 +0000 Subject: [PATCH 05/91] HDFS-3492. Fix some misuses of InputStream#skip. Contributed by Colin Patrick McCabe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1347192 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/lib/wsrs/InputStreamEntity.java | 5 +-- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../apache/hadoop/hdfs/BlockReaderLocal.java | 34 ++++--------------- .../hdfs/server/namenode/FSEditLogOp.java | 7 ++-- .../hdfs/TestShortCircuitLocalRead.java | 7 ++-- .../server/datanode/SimulatedFSDataset.java | 3 +- 6 files changed, 21 insertions(+), 38 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/InputStreamEntity.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/InputStreamEntity.java index e5361a80ef..21c25bd4c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/InputStreamEntity.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/InputStreamEntity.java @@ -42,10 +42,7 @@ public InputStreamEntity(InputStream is) { @Override public void write(OutputStream os) throws IOException { - long skipped = is.skip(offset); - if (skipped < offset) { - throw new IOException("Requested offset beyond stream size"); - } + IOUtils.skipFully(is, offset); if (len == -1) { IOUtils.copyBytes(is, os, 4096, true); } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 24b38eb2a3..03d8c8b679 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -310,6 +310,9 @@ Branch-2 ( Unreleased changes ) HDFS-3505. DirectoryScanner does not join all threads in shutdown. (Colin Patrick McCabe via eli) + HDFS-3492. Fix some misuses of InputStream#skip (Colin Patrick McCabe + via todd) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java index cd85ebae05..a51b710e29 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader; import org.apache.hadoop.hdfs.util.DirectBufferPool; import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.DataChecksum; @@ -315,23 +316,10 @@ private BlockReaderLocal(Configuration conf, String hdfsfile, boolean success = false; try { // Skip both input streams to beginning of the chunk containing startOffset - long toSkip = firstChunkOffset; - while (toSkip > 0) { - long skipped = dataIn.skip(toSkip); - if (skipped == 0) { - throw new IOException("Couldn't initialize input stream"); - } - toSkip -= skipped; - } + IOUtils.skipFully(dataIn, firstChunkOffset); if (checksumIn != null) { long checkSumOffset = (firstChunkOffset / bytesPerChecksum) * checksumSize; - while (checkSumOffset > 0) { - long skipped = checksumIn.skip(checkSumOffset); - if (skipped == 0) { - throw new IOException("Couldn't initialize checksum input stream"); - } - checkSumOffset -= skipped; - } + IOUtils.skipFully(dataIn, checkSumOffset); } success = true; } finally { @@ -636,17 +624,9 @@ public synchronized long skip(long n) throws IOException { slowReadBuff.position(slowReadBuff.limit()); checksumBuff.position(checksumBuff.limit()); - long dataSkipped = dataIn.skip(toskip); - if (dataSkipped != toskip) { - throw new IOException("skip error in data input stream"); - } - long checkSumOffset = (dataSkipped / bytesPerChecksum) * checksumSize; - if (checkSumOffset > 0) { - long skipped = checksumIn.skip(checkSumOffset); - if (skipped != checkSumOffset) { - throw new IOException("skip error in checksum input stream"); - } - } + IOUtils.skipFully(dataIn, toskip); + long checkSumOffset = (toskip / bytesPerChecksum) * checksumSize; + IOUtils.skipFully(checksumIn, checkSumOffset); // read into the middle of the chunk if (skipBuf == null) { @@ -701,4 +681,4 @@ public Socket takeSocket() { public boolean hasSentStatusCode() { return false; } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java index 489f030e13..2aa8f736c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java @@ -44,6 +44,7 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.ArrayWritable; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableFactories; @@ -2289,9 +2290,11 @@ public FSEditLogOp readOp(boolean skipBrokenEdits) throws IOException { // 0xff, we want to skip over that region, because there's nothing // interesting there. long numSkip = e.getNumAfterTerminator(); - if (in.skip(numSkip) < numSkip) { + try { + IOUtils.skipFully(in, numSkip); + } catch (IOException t) { FSImage.LOG.error("Failed to skip " + numSkip + " bytes of " + - "garbage after an OP_INVALID. Unexpected early EOF."); + "garbage after an OP_INVALID. Unexpected early EOF.", t); return null; } } catch (IOException e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java index 169756759e..9c37b7a2fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.junit.Assert; @@ -94,8 +95,7 @@ static void checkFileContent(FileSystem fs, Path name, byte[] expected, // Now read using a different API. actual = new byte[expected.length-readOffset]; stm = fs.open(name); - long skipped = stm.skip(readOffset); - Assert.assertEquals(skipped, readOffset); + IOUtils.skipFully(stm, readOffset); //Read a small number of bytes first. int nread = stm.read(actual, 0, 3); nread += stm.read(actual, nread, 2); @@ -123,8 +123,7 @@ static void checkFileContentDirect(FileSystem fs, Path name, byte[] expected, ByteBuffer actual = ByteBuffer.allocate(expected.length - readOffset); - long skipped = stm.skip(readOffset); - Assert.assertEquals(skipped, readOffset); + IOUtils.skipFully(stm, readOffset); actual.limit(3); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index e69b1c3021..d4a7b3a257 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.DiskChecker.DiskErrorException; @@ -686,7 +687,7 @@ synchronized InputStream getBlockInputStream(ExtendedBlock b public synchronized InputStream getBlockInputStream(ExtendedBlock b, long seekOffset) throws IOException { InputStream result = getBlockInputStream(b); - result.skip(seekOffset); + IOUtils.skipFully(result, seekOffset); return result; } From 216521037a6489a8f1ce09654a79574a608b2269 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Thu, 7 Jun 2012 17:56:04 +0000 Subject: [PATCH 06/91] svn merge -c -1347092 for reverting HADOOP-8368 again. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1347738 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 - hadoop-common-project/hadoop-common/pom.xml | 103 +++++-- .../hadoop-common/src/CMakeLists.txt | 131 --------- .../hadoop-common/src/config.h.cmake | 10 - .../src/main/native/.autom4te.cfg | 42 +++ .../hadoop-common/src/main/native/Makefile.am | 66 +++++ .../src/main/native/acinclude.m4 | 28 ++ .../src/main/native/configure.ac | 130 +++++++++ .../src/main/native/lib/Makefile.am | 47 +++ .../hadoop/io/compress/lz4/Lz4Compressor.c | 5 +- .../hadoop/io/compress/lz4/Lz4Decompressor.c | 5 +- .../io/compress/snappy/SnappyCompressor.c | 36 ++- .../io/compress/snappy/SnappyDecompressor.c | 36 ++- .../org_apache_hadoop_io_compress_snappy.h | 41 ++- .../hadoop/io/compress/zlib/Makefile.am | 53 ++++ .../hadoop/io/compress/zlib/ZlibCompressor.c | 32 ++- .../io/compress/zlib/ZlibDecompressor.c | 32 ++- .../zlib/org_apache_hadoop_io_compress_zlib.h | 39 ++- .../org/apache/hadoop/io/nativeio/NativeIO.c | 4 +- .../src/org/apache/hadoop/util/NativeCrc32.c | 4 +- .../src/main/native/src/org_apache_hadoop.h | 17 +- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 76 ++++- .../hadoop-hdfs/src/CMakeLists.txt | 123 -------- .../hadoop-hdfs/src/config.h.cmake | 6 - .../src/contrib/fuse-dfs/Makefile.am | 27 ++ .../src/contrib/fuse-dfs/acinclude.m4 | 270 ++++++++++++++++++ .../src/contrib/fuse-dfs/configure.ac | 82 ++++++ .../hadoop-hdfs/src/contrib/fuse-dfs/pom.xml | 164 +++++++++++ .../src/contrib/fuse-dfs/src/CMakeLists.txt | 73 ----- .../src/contrib/fuse-dfs/src/Makefile.am | 22 ++ .../src/contrib/fuse-dfs/src/fuse_dfs.h | 8 +- .../hadoop-hdfs/src/main/native/Makefile.am | 42 +++ .../hadoop-hdfs/src/main/native/configure.ac | 125 ++++++++ .../src/main/native/m4/apfunctions.m4 | 41 +++ .../hadoop-hdfs/src/main/native/m4/apjava.m4 | 142 +++++++++ .../src/main/native/m4/apsupport.m4 | 168 +++++++++++ hadoop-hdfs-project/pom.xml | 1 + .../hadoop-yarn-server-nodemanager/pom.xml | 64 +++-- .../src/CMakeLists.txt | 66 ----- .../src/config.h.cmake | 6 - .../native/container-executor/.autom4te.cfg | 42 +++ .../.deps/container-executor.Po | 1 + .../native/container-executor/Makefile.am | 32 +++ .../native/container-executor/configure.ac | 54 ++++ .../native/container-executor/impl/main.c | 5 +- 45 files changed, 1991 insertions(+), 512 deletions(-) delete mode 100644 hadoop-common-project/hadoop-common/src/CMakeLists.txt delete mode 100644 hadoop-common-project/hadoop-common/src/config.h.cmake create mode 100644 hadoop-common-project/hadoop-common/src/main/native/.autom4te.cfg create mode 100644 hadoop-common-project/hadoop-common/src/main/native/Makefile.am create mode 100644 hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 create mode 100644 hadoop-common-project/hadoop-common/src/main/native/configure.ac create mode 100644 hadoop-common-project/hadoop-common/src/main/native/lib/Makefile.am create mode 100644 hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/config.h.cmake create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/Makefile.am create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/acinclude.m4 create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/configure.ac create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/pom.xml delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/CMakeLists.txt create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/Makefile.am create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/Makefile.am create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/configure.ac create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apfunctions.m4 create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apjava.m4 create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apsupport.m4 delete mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt delete mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.autom4te.cfg create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.deps/container-executor.Po create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/Makefile.am create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/configure.ac diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 7c7fa5d85c..477e336c1e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -226,8 +226,6 @@ Branch-2 ( Unreleased changes ) HADOOP-8358. Config-related WARN for dfs.web.ugi can be avoided. (harsh) - HADOOP-8368. Use CMake rather than autotools to build native code (ccccabe via tucu) - HADOOP-8450. Remove src/test/system. (eli) BUG FIXES diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index c045a7fcdb..a36b74d1e0 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -536,10 +536,31 @@ /usr/local ${snappy.prefix}/lib ${snappy.prefix}/include - + + org.apache.maven.plugins + maven-antrun-plugin + + + compile + compile + + run + + + + + + + + + + + + + org.codehaus.mojo native-maven-plugin @@ -569,27 +590,73 @@ - org.apache.maven.plugins - maven-antrun-plugin + org.codehaus.mojo + make-maven-plugin - make + compile compile - run - - - - - - - - - - - - + + autoreconf + configure + make-install + + + + ${project.build.directory}/native + + -i + -f + + + + + + OS_NAME + ${os.name} + + + OS_ARCH + ${os.arch} + + + JVM_DATA_MODEL + ${sun.arch.data.model} + + + + CPPFLAGS=-I${snappy.include} + LDFLAGS=-L${snappy.lib} + + ${project.build.directory}/native + /usr/local + + + + + OS_NAME + ${os.name} + + + OS_ARCH + ${os.arch} + + + JVM_DATA_MODEL + ${sun.arch.data.model} + + + HADOOP_NATIVE_SRCDIR + ${project.build.directory}/native + + + + + ${project.build.directory}/native/target + + @@ -633,7 +700,7 @@ maven-antrun-plugin - kdc + compile compile run diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt deleted file mode 100644 index 127e2d9c20..0000000000 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ /dev/null @@ -1,131 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -cmake_minimum_required(VERSION 2.6 FATAL_ERROR) - -# Default to release builds -set(CMAKE_BUILD_TYPE, Release) - -# If JVM_ARCH_DATA_MODEL is 32, compile all binaries as 32-bit. -# This variable is set by maven. -if (JVM_ARCH_DATA_MODEL EQUAL 32) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") - set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32") -endif (JVM_ARCH_DATA_MODEL EQUAL 32) - -# Compile a library with both shared and static variants -function(add_dual_library LIBNAME) - add_library(${LIBNAME} SHARED ${ARGN}) - add_library(${LIBNAME}_static STATIC ${ARGN}) - set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME}) -endfunction(add_dual_library) - -# Link both a static and a dynamic target against some libraries -function(target_link_dual_libraries LIBNAME) - target_link_libraries(${LIBNAME} ${ARGN}) - target_link_libraries(${LIBNAME}_static ${ARGN}) -endfunction(target_link_dual_libraries) - -function(output_directory TGT DIR) - SET_TARGET_PROPERTIES(${TGT} PROPERTIES - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") - SET_TARGET_PROPERTIES(${TGT} PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") - SET_TARGET_PROPERTIES(${TGT} PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") -endfunction(output_directory TGT DIR) - -function(dual_output_directory TGT DIR) - output_directory(${TGT} "${DIR}") - output_directory(${TGT}_static "${DIR}") -endfunction(dual_output_directory TGT DIR) - -if (NOT GENERATED_JAVAH) - # Must identify where the generated headers have been placed - MESSAGE(FATAL_ERROR "You must set the cmake variable GENERATED_JAVAH") -endif (NOT GENERATED_JAVAH) -find_package(JNI REQUIRED) -find_package(ZLIB REQUIRED) - -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64") -set(D main/native/src/org/apache/hadoop) - -GET_FILENAME_COMPONENT(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME) - -INCLUDE(CheckFunctionExists) -INCLUDE(CheckCSourceCompiles) -CHECK_FUNCTION_EXISTS(sync_file_range HAVE_SYNC_FILE_RANGE) -CHECK_FUNCTION_EXISTS(posix_fadvise HAVE_POSIX_FADVISE) - -find_library(SNAPPY_LIBRARY NAMES snappy PATHS) -find_path(SNAPPY_INCLUDE_DIR NAMES snappy.h PATHS) -if (SNAPPY_LIBRARY) - GET_FILENAME_COMPONENT(HADOOP_SNAPPY_LIBRARY ${SNAPPY_LIBRARY} NAME) - set(SNAPPY_SOURCE_FILES - "${D}/io/compress/snappy/SnappyCompressor.c" - "${D}/io/compress/snappy/SnappyDecompressor.c") -else (${SNAPPY_LIBRARY}) - set(SNAPPY_INCLUDE_DIR "") - set(SNAPPY_SOURCE_FILES "") -endif (SNAPPY_LIBRARY) - -include_directories( - ${GENERATED_JAVAH} - main/native/src - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/src - ${CMAKE_BINARY_DIR} - ${JNI_INCLUDE_DIRS} - ${ZLIB_INCLUDE_DIRS} - ${SNAPPY_INCLUDE_DIR} -) -CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h) - -add_dual_library(hadoop - ${D}/io/compress/lz4/Lz4Compressor.c - ${D}/io/compress/lz4/Lz4Decompressor.c - ${D}/io/compress/lz4/lz4.c - ${SNAPPY_SOURCE_FILES} - ${D}/io/compress/zlib/ZlibCompressor.c - ${D}/io/compress/zlib/ZlibDecompressor.c - ${D}/io/nativeio/NativeIO.c - ${D}/io/nativeio/errno_enum.c - ${D}/io/nativeio/file_descriptor.c - ${D}/security/JniBasedUnixGroupsMapping.c - ${D}/security/JniBasedUnixGroupsNetgroupMapping.c - ${D}/security/getGroup.c - ${D}/util/NativeCrc32.c - ${D}/util/bulk_crc32.c -) -target_link_dual_libraries(hadoop - dl - ${JAVA_JVM_LIBRARY} -) -SET(LIBHADOOP_VERSION "1.0.0") -SET_TARGET_PROPERTIES(hadoop PROPERTIES - SOVERSION ${LIBHADOOP_VERSION}) -dual_output_directory(hadoop target/usr/local/lib) - -if (HADOOP_RUNAS_HOME) - add_executable(runAs - test/system/c++/runAs/runAs.c - test/system/c++/runAs/main.c - ) - output_directory(runAs target/usr/local/bin) -endif (HADOOP_RUNAS_HOME) diff --git a/hadoop-common-project/hadoop-common/src/config.h.cmake b/hadoop-common-project/hadoop-common/src/config.h.cmake deleted file mode 100644 index 9098b68b87..0000000000 --- a/hadoop-common-project/hadoop-common/src/config.h.cmake +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef CONFIG_H -#define CONFIG_H - -#cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@" -#cmakedefine HADOOP_RUNAS_HOME "@HADOOP_RUNAS_HOME@" -#cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@" -#cmakedefine HAVE_SYNC_FILE_RANGE -#cmakedefine HAVE_POSIX_FADVISE - -#endif diff --git a/hadoop-common-project/hadoop-common/src/main/native/.autom4te.cfg b/hadoop-common-project/hadoop-common/src/main/native/.autom4te.cfg new file mode 100644 index 0000000000..a69c197883 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/.autom4te.cfg @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# autom4te configuration for hadoop-native library +# + +begin-language: "Autoheader-preselections" +args: --no-cache +end-language: "Autoheader-preselections" + +begin-language: "Automake-preselections" +args: --no-cache +end-language: "Automake-preselections" + +begin-language: "Autoreconf-preselections" +args: --no-cache +end-language: "Autoreconf-preselections" + +begin-language: "Autoconf-without-aclocal-m4" +args: --no-cache +end-language: "Autoconf-without-aclocal-m4" + +begin-language: "Autoconf" +args: --no-cache +end-language: "Autoconf" + diff --git a/hadoop-common-project/hadoop-common/src/main/native/Makefile.am b/hadoop-common-project/hadoop-common/src/main/native/Makefile.am new file mode 100644 index 0000000000..c4ca564c2b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/Makefile.am @@ -0,0 +1,66 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Notes: +# 1. This makefile is designed to do the actual builds in $(HADOOP_PREFIX)/build/native/${os.name}-${os-arch}. +# 2. This makefile depends on the following environment variables to function correctly: +# * HADOOP_NATIVE_SRCDIR +# * JAVA_HOME +# * JVM_DATA_MODEL +# * OS_NAME +# * OS_ARCH +# All these are setup by build.xml. +# + +# Export $(PLATFORM) to prevent proliferation of sub-shells +export PLATFORM = $(shell echo $$OS_NAME | tr [A-Z] [a-z]) + +ACLOCAL_AMFLAGS = -I m4 +AM_CPPFLAGS = @JNI_CPPFLAGS@ -I$(HADOOP_NATIVE_SRCDIR)/src \ + -I$(HADOOP_NATIVE_SRCDIR)/javah +AM_LDFLAGS = @JNI_LDFLAGS@ +AM_CFLAGS = -g -Wall -fPIC -O2 +if SPECIFY_DATA_MODEL +AM_LDFLAGS += -m$(JVM_DATA_MODEL) +AM_CFLAGS += -m$(JVM_DATA_MODEL) +endif + +lib_LTLIBRARIES = libhadoop.la +libhadoop_la_SOURCES = src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c \ + src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c \ + src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c \ + src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c \ + src/org/apache/hadoop/io/compress/lz4/lz4.c \ + src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c \ + src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c \ + src/org/apache/hadoop/security/getGroup.c \ + src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c \ + src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c \ + src/org/apache/hadoop/io/nativeio/file_descriptor.c \ + src/org/apache/hadoop/io/nativeio/errno_enum.c \ + src/org/apache/hadoop/io/nativeio/NativeIO.c \ + src/org/apache/hadoop/util/NativeCrc32.c \ + src/org/apache/hadoop/util/bulk_crc32.c + +libhadoop_la_LDFLAGS = -version-info 1:0:0 $(AM_LDFLAGS) +libhadoop_la_LIBADD = -ldl -ljvm + +# +#vim: sw=4: ts=4: noet +# diff --git a/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 b/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 new file mode 100644 index 0000000000..93e05b8148 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 @@ -0,0 +1,28 @@ +# AC_COMPUTE_NEEDED_DSO(LIBRARY, TEST_PROGRAM, PREPROC_SYMBOL) +# -------------------------------------------------- +# Compute the 'actual' dynamic-library used +# for LIBRARY and set it to PREPROC_SYMBOL +AC_DEFUN([AC_COMPUTE_NEEDED_DSO], +[ +AC_CACHE_CHECK([Checking for the 'actual' dynamic-library for '-l$1'], ac_cv_libname_$1, + [ + echo '$2' > conftest.c + if test -z "`${CC} ${LDFLAGS} -o conftest conftest.c -l$1 2>&1`"; then + dnl Try objdump and ldd in that order to get the dynamic library + if test ! -z "`which objdump | grep -v 'no objdump'`"; then + ac_cv_libname_$1="`objdump -p conftest | grep NEEDED | grep $1 | sed 's/\W*NEEDED\W*\(.*\)\W*$/\"\1\"/'`" + elif test ! -z "`which ldd | grep -v 'no ldd'`"; then + ac_cv_libname_$1="`ldd conftest | grep $1 | sed 's/^[[[^A-Za-z0-9]]]*\([[[A-Za-z0-9\.]]]*\)[[[^A-Za-z0-9]]]*=>.*$/\"\1\"/'`" + elif test ! -z "`which otool | grep -v 'no otool'`"; then + ac_cv_libname_$1=\"`otool -L conftest | grep $1 | sed -e 's/^[ ]*//' -e 's/ .*//' -e 's/.*\/\(.*\)$/\1/'`\"; + else + AC_MSG_ERROR(Can't find either 'objdump' or 'ldd' or 'otool' to compute the dynamic library for '-l$1') + fi + else + ac_cv_libname_$1=libnotfound.so + fi + rm -f conftest* + ] +) +AC_DEFINE_UNQUOTED($3, ${ac_cv_libname_$1}, [The 'actual' dynamic-library for '-l$1']) +])# AC_COMPUTE_NEEDED_DSO diff --git a/hadoop-common-project/hadoop-common/src/main/native/configure.ac b/hadoop-common-project/hadoop-common/src/main/native/configure.ac new file mode 100644 index 0000000000..34408d6418 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/configure.ac @@ -0,0 +1,130 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# configure.ac for hadoop native code. +# + +# Notes: +# 1. This configure.ac depends on the following environment variables to function correctly: +# * HADOOP_NATIVE_SRCDIR +# * JAVA_HOME +# * JVM_DATA_MODEL +# * OS_NAME +# * OS_ARCH +# All these are setup by build.xml. + +# -*- Autoconf -*- +# Process this file with autoconf to produce a configure script. +# + +AC_PREREQ(2.59) +AC_INIT(src/org_apache_hadoop.h) +AC_CONFIG_SRCDIR([src/org_apache_hadoop.h]) +AC_CONFIG_AUX_DIR([config]) +AC_CONFIG_MACRO_DIR([m4]) +AC_CONFIG_HEADER([config.h]) +AC_SYS_LARGEFILE +AC_GNU_SOURCE + +AM_INIT_AUTOMAKE(hadoop,1.0.0) + +# Checks for programs. +AC_PROG_CC +AC_PROG_LIBTOOL + +# Checks for libraries. +dnl Check for '-ldl' +AC_CHECK_LIB([dl], [dlopen]) + +dnl Check for '-ljvm' +JNI_LDFLAGS="" +if test $JAVA_HOME != "" +then + JNI_LDFLAGS="-L$JAVA_HOME/jre/lib/$OS_ARCH/server" + JVMSOPATH=`find $JAVA_HOME/jre/ -name libjvm.so | head -n 1` + JNI_LDFLAGS="$JNI_LDFLAGS -L`dirname $JVMSOPATH`" +fi +LDFLAGS="$LDFLAGS $JNI_LDFLAGS" +AC_CHECK_LIB([jvm], [JNI_GetCreatedJavaVMs]) +AC_SUBST([JNI_LDFLAGS]) + +# Checks for header files. +dnl Check for Ansi C headers +AC_HEADER_STDC + +dnl Check for other standard C headers +AC_CHECK_HEADERS([stdio.h stddef.h], [], AC_MSG_ERROR(Some system headers not found... please ensure their presence on your platform.)) + +dnl Check for JNI headers +JNI_CPPFLAGS="" +if test $JAVA_HOME != "" +then + for dir in `find $JAVA_HOME/include -follow -type d` + do + JNI_CPPFLAGS="$JNI_CPPFLAGS -I$dir" + done +fi +cppflags_bak=$CPPFLAGS +CPPFLAGS="$CPPFLAGS $JNI_CPPFLAGS" +AC_CHECK_HEADERS([jni.h], [], AC_MSG_ERROR([Native java headers not found. Is \$JAVA_HOME set correctly?])) +CPPFLAGS=$cppflags_bak +AC_SUBST([JNI_CPPFLAGS]) + +dnl Check for zlib headers +AC_CHECK_HEADERS([zlib.h zconf.h], + AC_COMPUTE_NEEDED_DSO(z, + [#include "zlib.h" + int main(int argc, char **argv){zlibVersion();return 0;}], + HADOOP_ZLIB_LIBRARY), + AC_MSG_ERROR(Zlib headers were not found... native-hadoop library needs zlib to build. Please install the requisite zlib development package.)) + +dnl Check for snappy headers +AC_CHECK_HEADERS([snappy-c.h], + AC_COMPUTE_NEEDED_DSO(snappy, + [#include "snappy-c.h" + int main(int argc, char **argv){snappy_compress(0,0,0,0);return 0;}], + HADOOP_SNAPPY_LIBRARY), + AC_MSG_WARN(Snappy headers were not found... building without snappy.)) + +dnl Check for headers needed by the native Group resolution implementation +AC_CHECK_HEADERS([fcntl.h stdlib.h string.h unistd.h], [], AC_MSG_ERROR(Some system headers not found... please ensure their presence on your platform.)) + +dnl check for posix_fadvise +AC_CHECK_HEADERS(fcntl.h, [AC_CHECK_FUNCS(posix_fadvise)]) + +dnl check for sync_file_range +AC_CHECK_HEADERS(fcntl.h, [AC_CHECK_FUNCS(sync_file_range)]) + +# Checks for typedefs, structures, and compiler characteristics. +AC_C_CONST + +# Checks for library functions. +AC_CHECK_FUNCS([memset]) + +# Check for nonstandard STRERROR_R +AC_FUNC_STRERROR_R + +AM_CONDITIONAL([SPECIFY_DATA_MODEL], [case $host_cpu in arm*) false;; *) true;; esac]) + +AC_CONFIG_FILES([Makefile]) +AC_OUTPUT + +# +#vim: sw=2: ts=2: noet +# diff --git a/hadoop-common-project/hadoop-common/src/main/native/lib/Makefile.am b/hadoop-common-project/hadoop-common/src/main/native/lib/Makefile.am new file mode 100644 index 0000000000..9b536ff440 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/lib/Makefile.am @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Makefile template for building libhadoop.so +# + +# +# Notes: +# 1. This makefile is designed to do the actual builds in $(HADOOP_PREFIX)/build/native/${os.name}-${os.arch}/lib +# 2. This makefile depends on the following environment variables to function correctly: +# * HADOOP_NATIVE_SRCDIR +# * JAVA_HOME +# * OS_ARCH +# All these are setup by build.xml and/or the top-level makefile. +# + +# Add .lo files in $(SUBDIRS) to construct libhadoop.so +HADOOP_OBJS = $(foreach path,$(addprefix ../,$(SUBDIRS)),$(wildcard $(path)/*.lo)) +AM_LDFLAGS = @JNI_LDFLAGS@ +if SPECIFY_DATA_MODEL +AM_LDFLAGS += -m$(JVM_DATA_MODEL) +endif + +lib_LTLIBRARIES = libhadoop.la +libhadoop_la_SOURCES = +libhadoop_la_LDFLAGS = -version-info 1:0:0 $(AM_LDFLAGS) +libhadoop_la_LIBADD = $(HADOOP_OBJS) -ldl -ljvm + +# +#vim: sw=4: ts=4: noet +# diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c index 641ecd73b7..d52a4f6b2a 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c @@ -16,7 +16,10 @@ * limitations under the License. */ -#include "config.h" +#if defined HAVE_CONFIG_H + #include +#endif + #include "org_apache_hadoop.h" #include "org_apache_hadoop_io_compress_lz4_Lz4Compressor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c index 3eebc1859d..547b027cc1 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c @@ -16,7 +16,10 @@ * limitations under the License. */ -#include "config.h" +#if defined HAVE_CONFIG_H + #include +#endif + #include "org_apache_hadoop.h" #include "org_apache_hadoop_io_compress_lz4_Lz4Decompressor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c index 96a2402ae7..13991c23f4 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c @@ -16,12 +16,36 @@ * limitations under the License. */ -#include -#include -#include -#include +#if defined HAVE_CONFIG_H + #include +#endif + +#if defined HADOOP_SNAPPY_LIBRARY + +#if defined HAVE_STDIO_H + #include +#else + #error 'stdio.h not found' +#endif + +#if defined HAVE_STDLIB_H + #include +#else + #error 'stdlib.h not found' +#endif + +#if defined HAVE_STRING_H + #include +#else + #error 'string.h not found' +#endif + +#if defined HAVE_DLFCN_H + #include +#else + #error 'dlfcn.h not found' +#endif -#include "config.h" #include "org_apache_hadoop_io_compress_snappy.h" #include "org_apache_hadoop_io_compress_snappy_SnappyCompressor.h" @@ -99,3 +123,5 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso return (jint)compressed_direct_buf_len; } + +#endif //define HADOOP_SNAPPY_LIBRARY diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c index a5f07ca556..767c5f4b31 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c @@ -16,12 +16,36 @@ * limitations under the License. */ -#include -#include -#include -#include +#if defined HAVE_CONFIG_H + #include +#endif + +#if defined HADOOP_SNAPPY_LIBRARY + +#if defined HAVE_STDIO_H + #include +#else + #error 'stdio.h not found' +#endif + +#if defined HAVE_STDLIB_H + #include +#else + #error 'stdlib.h not found' +#endif + +#if defined HAVE_STRING_H + #include +#else + #error 'string.h not found' +#endif + +#if defined HAVE_DLFCN_H + #include +#else + #error 'dlfcn.h not found' +#endif -#include "config.h" #include "org_apache_hadoop_io_compress_snappy.h" #include "org_apache_hadoop_io_compress_snappy_SnappyDecompressor.h" @@ -103,3 +127,5 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyDecompres return (jint)uncompressed_direct_buf_len; } + +#endif //define HADOOP_SNAPPY_LIBRARY diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h index 3e99d5d20d..815e030673 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h @@ -17,13 +17,42 @@ */ -#ifndef ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H +#if !defined ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H #define ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H -#include "org_apache_hadoop.h" -#include -#include -#include -#include + +#if defined HAVE_CONFIG_H + #include +#endif + +#if defined HADOOP_SNAPPY_LIBRARY + + #if defined HAVE_STDDEF_H + #include + #else + #error 'stddef.h not found' + #endif + + #if defined HAVE_SNAPPY_C_H + #include + #else + #error 'Please install snappy-development packages for your platform.' + #endif + + #if defined HAVE_DLFCN_H + #include + #else + #error "dlfcn.h not found" + #endif + + #if defined HAVE_JNI_H + #include + #else + #error 'jni.h not found' + #endif + + #include "org_apache_hadoop.h" + +#endif //define HADOOP_SNAPPY_LIBRARY #endif //ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am new file mode 100644 index 0000000000..821f33f052 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Makefile template for building native 'zlib' for hadoop. +# + +# +# Notes: +# 1. This makefile is designed to do the actual builds in $(HADOOP_PREFIX)/build/native/${os.name}-${os.arch}/$(subdir) . +# 2. This makefile depends on the following environment variables to function correctly: +# * HADOOP_NATIVE_SRCDIR +# * JAVA_HOME +# * JVM_DATA_MODEL +# * OS_ARCH +# * PLATFORM +# All these are setup by build.xml and/or the top-level makefile. +# 3. The creation of requisite jni headers/stubs are also done by build.xml and they are +# assumed to be in $(HADOOP_PREFIX)/build/native/src/org/apache/hadoop/io/compress/zlib. +# + +# The 'vpath directive' to locate the actual source files +vpath %.c $(HADOOP_NATIVE_SRCDIR)/$(subdir) + +AM_CPPFLAGS = @JNI_CPPFLAGS@ -I$(HADOOP_NATIVE_SRCDIR)/src +AM_LDFLAGS = @JNI_LDFLAGS@ +AM_CFLAGS = -g -Wall -fPIC -O2 +if SPECIFY_DATA_MODEL +AM_CFLAGS += -m$(JVM_DATA_MODEL) +endif + +noinst_LTLIBRARIES = libnativezlib.la +libnativezlib_la_SOURCES = ZlibCompressor.c ZlibDecompressor.c +libnativezlib_la_LIBADD = -ldl -ljvm + +# +#vim: sw=4: ts=4: noet +# diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c index 689c783ef7..9ada3f03b0 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c @@ -16,12 +16,34 @@ * limitations under the License. */ -#include -#include -#include -#include +#if defined HAVE_CONFIG_H + #include +#endif + +#if defined HAVE_STDIO_H + #include +#else + #error 'stdio.h not found' +#endif + +#if defined HAVE_STDLIB_H + #include +#else + #error 'stdlib.h not found' +#endif + +#if defined HAVE_STRING_H + #include +#else + #error 'string.h not found' +#endif + +#if defined HAVE_DLFCN_H + #include +#else + #error 'dlfcn.h not found' +#endif -#include "config.h" #include "org_apache_hadoop_io_compress_zlib.h" #include "org_apache_hadoop_io_compress_zlib_ZlibCompressor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c index 6abe36381f..3047dba267 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c @@ -16,12 +16,34 @@ * limitations under the License. */ -#include -#include -#include -#include +#if defined HAVE_CONFIG_H + #include +#endif + +#if defined HAVE_STDIO_H + #include +#else + #error 'stdio.h not found' +#endif + +#if defined HAVE_STDLIB_H + #include +#else + #error 'stdlib.h not found' +#endif + +#if defined HAVE_STRING_H + #include +#else + #error 'string.h not found' +#endif + +#if defined HAVE_DLFCN_H + #include +#else + #error 'dlfcn.h not found' +#endif -#include "config.h" #include "org_apache_hadoop_io_compress_zlib.h" #include "org_apache_hadoop_io_compress_zlib_ZlibDecompressor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h index c53aa531c9..16b607b4a9 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h @@ -19,13 +19,40 @@ #if !defined ORG_APACHE_HADOOP_IO_COMPRESS_ZLIB_ZLIB_H #define ORG_APACHE_HADOOP_IO_COMPRESS_ZLIB_ZLIB_H -#include -#include -#include -#include -#include +#if defined HAVE_CONFIG_H + #include +#endif + +#if defined HAVE_STDDEF_H + #include +#else + #error 'stddef.h not found' +#endif + +#if defined HAVE_ZLIB_H + #include +#else + #error 'Please install zlib-development packages for your platform.' +#endif + +#if defined HAVE_ZCONF_H + #include +#else + #error 'Please install zlib-development packages for your platform.' +#endif + +#if defined HAVE_DLFCN_H + #include +#else + #error "dlfcn.h not found" +#endif + +#if defined HAVE_JNI_H + #include +#else + #error 'jni.h not found' +#endif -#include "config.h" #include "org_apache_hadoop.h" /* A helper macro to convert the java 'stream-handle' to a z_stream pointer. */ diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index c08ea037d9..fbcf9563ee 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -16,6 +16,9 @@ * limitations under the License. */ +// get the autoconf settings +#include "config.h" + #include #include #include @@ -29,7 +32,6 @@ #include #include -#include "config.h" #include "org_apache_hadoop.h" #include "org_apache_hadoop_io_nativeio_NativeIO.h" #include "file_descriptor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c index dd51c0a257..869c2ba2e8 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c @@ -16,6 +16,9 @@ * limitations under the License. */ +// get the autoconf settings +#include "config.h" + #include #include #include @@ -23,7 +26,6 @@ #include #include -#include "config.h" #include "org_apache_hadoop.h" #include "org_apache_hadoop_util_NativeCrc32.h" #include "gcc_optimizations.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h b/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h index a50c41dbbb..7a777c2f4f 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h @@ -24,10 +24,21 @@ #if !defined ORG_APACHE_HADOOP_H #define ORG_APACHE_HADOOP_H -#include -#include +#if defined HAVE_CONFIG_H + #include +#endif + +#if defined HAVE_DLFCN_H + #include +#else + #error "dlfcn.h not found" +#endif -#include "config.h" +#if defined HAVE_JNI_H + #include +#else + #error 'jni.h not found' +#endif /* A helper macro to 'throw' a java exception. */ #define THROW(env, exception_name, message) \ diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index c775c51e3f..ebff0f57f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -415,22 +415,76 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> maven-antrun-plugin - make + compile compile - run + + run + - - - - - - - + + + + + + + + org.codehaus.mojo + make-maven-plugin + + + compile + compile + + autoreconf + configure + make-install + + + + ${project.build.directory}/native + + -i + -f + + + + + + ac_cv_func_malloc_0_nonnull + yes + + + JVM_ARCH + ${sun.arch.data.model} + + + + + ${project.build.directory}/native + /usr/local + + + + + ac_cv_func_malloc_0_nonnull + yes + + + JVM_ARCH + ${sun.arch.data.model} + + + + + ${project.build.directory}/native/target + + + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.0.0-SNAPSHOT + ../../../../../hadoop-project + + org.apache.hadoop.contrib + hadoop-hdfs-fuse + 3.0.0-SNAPSHOT + pom + + Apache Hadoop HDFS Fuse + Apache Hadoop HDFS Fuse + + + + org.apache.hadoop + hadoop-hdfs + compile + + + org.apache.hadoop + hadoop-hdfs + test + test-jar + + + + + + + + + org.apache.maven.plugins + maven-eclipse-plugin + 2.6 + + + org.apache.maven.plugins + maven-surefire-plugin + + 1 + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + javadoc + + site + + true + true + false + ${maven.compile.source} + ${maven.compile.encoding} + + + HttpFs API + * + + + + + + + + org.apache.maven.plugins + maven-project-info-reports-plugin + + + + false + + + dependencies + + site + + + + + org.apache.rat + apache-rat-plugin + + + + + + + + + + + fuse + + false + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + prepare-compile-native + generate-sources + + run + + + + + + + + + + + compile-fuse + compile + + run + + + + + + + + + + + + + + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/CMakeLists.txt deleted file mode 100644 index fb3c580e94..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/CMakeLists.txt +++ /dev/null @@ -1,73 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Find Linux FUSE -IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") - find_package(PkgConfig REQUIRED) - pkg_check_modules(FUSE fuse) - IF(FUSE_FOUND) - FLATTEN_LIST("${FUSE_CFLAGS}" " " FUSE_CFLAGS) - FLATTEN_LIST("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FUSE_CFLAGS}") - set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} ${FUSE_LDFLAGS}") - MESSAGE(STATUS "Building Linux FUSE client.") - include_directories(${FUSE_INCLUDE_DIRS}) - ELSE(FUSE_FOUND) - MESSAGE(STATUS "Failed to find Linux FUSE libraries or include files. Will not build FUSE client.") - ENDIF(FUSE_FOUND) -ELSE (${CMAKE_SYSTEM_NAME} MATCHES "Linux") - MESSAGE(STATUS "Non-Linux system detected. Will not build FUSE client.") -ENDIF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") - -IF(FUSE_FOUND) - add_executable(fuse_dfs - fuse_dfs.c - fuse_options.c - fuse_connect.c - fuse_impls_access.c - fuse_impls_chmod.c - fuse_impls_chown.c - fuse_impls_create.c - fuse_impls_flush.c - fuse_impls_getattr.c - fuse_impls_mkdir.c - fuse_impls_mknod.c - fuse_impls_open.c - fuse_impls_read.c - fuse_impls_readdir.c - fuse_impls_release.c - fuse_impls_rename.c - fuse_impls_rmdir.c - fuse_impls_statfs.c - fuse_impls_symlink.c - fuse_impls_truncate.c - fuse_impls_unlink.c - fuse_impls_utimens.c - fuse_impls_write.c - fuse_init.c - fuse_stat_struct.c - fuse_trash.c - fuse_users.c - ) - target_link_libraries(fuse_dfs - ${FUSE_LIBRARIES} - ${JAVA_JVM_LIBRARY} - hdfs - m - ) -ENDIF(FUSE_FOUND) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/Makefile.am b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/Makefile.am new file mode 100644 index 0000000000..706297f314 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/Makefile.am @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +bin_PROGRAMS = fuse_dfs +fuse_dfs_SOURCES = fuse_dfs.c fuse_options.c fuse_trash.c fuse_stat_struct.c fuse_users.c fuse_init.c fuse_connect.c fuse_impls_access.c fuse_impls_chmod.c fuse_impls_chown.c fuse_impls_create.c fuse_impls_flush.c fuse_impls_getattr.c fuse_impls_mkdir.c fuse_impls_mknod.c fuse_impls_open.c fuse_impls_read.c fuse_impls_release.c fuse_impls_readdir.c fuse_impls_rename.c fuse_impls_rmdir.c fuse_impls_statfs.c fuse_impls_symlink.c fuse_impls_truncate.c fuse_impls_utimens.c fuse_impls_unlink.c fuse_impls_write.c +AM_CFLAGS= -Wall -g +AM_CPPFLAGS= -DPERMS=$(PERMS) -D_FILE_OFFSET_BITS=64 -I$(JAVA_HOME)/include -I$(HADOOP_PREFIX)/../../src/main/native -I$(JAVA_HOME)/include/linux -D_FUSE_DFS_VERSION=\"$(PACKAGE_VERSION)\" -DPROTECTED_PATHS=\"$(PROTECTED_PATHS)\" -I$(FUSE_HOME)/include +AM_LDFLAGS= -L$(HADOOP_PREFIX)/../../target/native/target/usr/local/lib64 -L$(HADOOP_PREFIX)/../../target/native/target/usr/local/lib -L$(FUSE_HOME)/lib -L$(JAVA_HOME)/jre/lib/$(OS_ARCH)/server +fuse_dfs_LDADD=-lfuse -lhdfs -ljvm -lm diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_dfs.h b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_dfs.h index 4554dbdbea..56ed9cb173 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_dfs.h +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_dfs.h @@ -31,9 +31,13 @@ #include #include -#include +#ifdef HAVE_CONFIG_H +#include +#endif -#include "config.h" +#ifdef HAVE_SETXATTR +#include +#endif // // Check if a path is in the mount option supplied protected paths. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/Makefile.am b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/Makefile.am new file mode 100644 index 0000000000..8bbd627315 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/Makefile.am @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +@PRODUCT_MK@ + +#AM_CPPFLAGS = -I$(top_srcdir) +ACLOCAL_AMFLAGS = -I m4 + +lib_LTLIBRARIES = libhdfs.la +libhdfs_la_SOURCES = hdfs.c hdfsJniHelper.c hdfs.h + +#check_PROGRAMS = hdfs_test hdfs_read hdfs_write +check_PROGRAMS = hdfs_test hdfs_read hdfs_write + +hdfs_test_SOURCES = hdfs_test.c hdfs.h +hdfs_test_LDADD = ${libdir}/libhdfs.la + +hdfs_read_SOURCES = hdfs_read.c +hdfs_read_LDADD = ${libdir}/libhdfs.la + +hdfs_write_SOURCES = hdfs_write.c +hdfs_write_LDADD = ${libdir}/libhdfs.la + +test: hdfs_test hdfs_read hdfs_write + ${LIBHDFS_SRC_DIR}/tests/test-libhdfs.sh + + +# vim: sw=4: ts=4: noet diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/configure.ac b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/configure.ac new file mode 100644 index 0000000000..d801fc4738 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/configure.ac @@ -0,0 +1,125 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Autoconf input file +# $Id$ + +AC_INIT([libhdfs], [0.1.0], omalley@apache.org) +AC_PREFIX_DEFAULT([`pwd`/../install]) +AC_CONFIG_AUX_DIR([config]) + +# Generates Makefile from Makefile.am. Modify when new subdirs are added. +# Change Makefile.am also to add subdirectly. +AM_INIT_AUTOMAKE(foreign no-dist) +AC_CONFIG_FILES(Makefile) + +LT_INIT + +AC_CONFIG_MACRO_DIR([m4]) +dnl ------------------------------------------------------------------------- +dnl Check current host (forget about cross compilation) and validate it +dnl against the cache (fail if the cache differs) +dnl ------------------------------------------------------------------------- +AP_MSG_HEADER([Current host]) +AC_CANONICAL_HOST() +AP_CANONICAL_HOST_CHECK() + +dnl ------------------------------------------------------------------------- +dnl Check C environment +dnl ------------------------------------------------------------------------- +AP_MSG_HEADER([C-Language compilation tools]) +AC_PROG_CC() +AC_CHECK_TOOL(RANLIB, ranlib, :) + +dnl ------------------------------------------------------------------------- +dnl Check if this host is supported +dnl ------------------------------------------------------------------------- +AP_MSG_HEADER([Host support]) +AP_SUPPORTED_HOST() +if test "$supported_os" = "darwin" +then + if test -z "$JAVA_HOME" -a -d /System/Library/Frameworks/JavaVM.framework/Home; then + JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Home + fi + + _prevdir=`/bin/pwd` + if test -n "$JAVA_HOME" -a -d "$JAVA_HOME/include"; then + cd "$JAVA_HOME/include" + elif test -n "$JAVA_HOME" -a -d "$JAVA_HOME/../Headers"; then + cd "$JAVA_HOME/../Headers" + else + cd /System/Library/Frameworks/JavaVM.framework/Headers + fi + CFLAGS="$CFLAGS -m${JVM_ARCH} -I`/bin/pwd -P`" + cd $_prevdir + unset _prevdir +fi + +dnl ------------------------------------------------------------------------- +dnl Check JAVA environment +dnl ------------------------------------------------------------------------- +AP_MSG_HEADER([Java compilation tools]) +AP_JAVA() +AP_SABLEVM() +AP_KAFFE() +AP_PROG_JAVAC() +AP_PROG_JAR() +AP_JVM_LIBDIR() +if test "$supported_os" != "darwin" +then + case $host_cpu in + arm*) ;; + *) + CFLAGS="$CFLAGS -m${JVM_ARCH}" + LDFLAGS="$LDFLAGS -m${JVM_ARCH}" + ;; + esac + AC_MSG_RESULT([VALUE OF JVM_ARCH IS :$JVM_ARCH]) + CFLAGS="$CFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/$supported_os" + LDFLAGS="$LDFLAGS -L$LIB_JVM_DIR -ljvm -Wl,-x" +fi + +dnl ------------------------------------------------------------------------- +dnl Add gcc specific CFLAGS. +dnl ------------------------------------------------------------------------- +if test "$GCC" = "yes" +then + CFLAGS="$CFLAGS -Wall -Wstrict-prototypes" + AC_MSG_RESULT([gcc flags added]) +fi +dnl ------------------------------------------------------------------------- +dnl Add gcc specific CFLAGS. +dnl ------------------------------------------------------------------------- +if test -z "$LDCMD" +then + LDCMD="$CC" +fi +AC_SUBST(LDCMD) + + +AC_PROG_CC +AC_PROG_LIBTOOL + +AC_TYPE_SIZE_T +AC_CHECK_FUNCS([strdup strerror strtoul]) +AC_CHECK_HEADERS([fcntl.h]) +AC_C_CONST +AC_C_VOLATILE +#AC_FUNC_MALLOC +AC_HEADER_STDBOOL +AC_SUBST(PRODUCT_MK) +AC_OUTPUT diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apfunctions.m4 b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apfunctions.m4 new file mode 100644 index 0000000000..cb5938ffca --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apfunctions.m4 @@ -0,0 +1,41 @@ +dnl +dnl Licensed to the Apache Software Foundation (ASF) under one or more +dnl contributor license agreements. See the NOTICE file distributed with +dnl this work for additional information regarding copyright ownership. +dnl The ASF licenses this file to You under the Apache License, Version 2.0 +dnl (the "License"); you may not use this file except in compliance with +dnl the License. You may obtain a copy of the License at +dnl +dnl http://www.apache.org/licenses/LICENSE-2.0 +dnl +dnl Unless required by applicable law or agreed to in writing, software +dnl distributed under the License is distributed on an "AS IS" BASIS, +dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +dnl See the License for the specific language governing permissions and +dnl limitations under the License. +dnl + +dnl ------------------------------------------------------------------------- +dnl Author Pier Fumagalli +dnl Version $Id$ +dnl ------------------------------------------------------------------------- + +AC_DEFUN([AP_MSG_HEADER],[ + printf "*** %s ***\n" "$1" 1>&2 + AC_PROVIDE([$0]) +]) + +AC_DEFUN([AP_CANONICAL_HOST_CHECK],[ + AC_MSG_CHECKING([cached host system type]) + if { test x"${ac_cv_host_system_type+set}" = x"set" && + test x"$ac_cv_host_system_type" != x"$host" ; } + then + AC_MSG_RESULT([$ac_cv_host_system_type]) + AC_MSG_ERROR([remove the \"$cache_file\" file and re-run configure]) + else + AC_MSG_RESULT(ok) + ac_cv_host_system_type="$host" + fi + AC_PROVIDE([$0]) +]) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apjava.m4 b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apjava.m4 new file mode 100644 index 0000000000..993fc5bed9 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apjava.m4 @@ -0,0 +1,142 @@ +dnl +dnl Licensed to the Apache Software Foundation (ASF) under one or more +dnl contributor license agreements. See the NOTICE file distributed with +dnl this work for additional information regarding copyright ownership. +dnl The ASF licenses this file to You under the Apache License, Version 2.0 +dnl (the "License"); you may not use this file except in compliance with +dnl the License. You may obtain a copy of the License at +dnl +dnl http://www.apache.org/licenses/LICENSE-2.0 +dnl +dnl Unless required by applicable law or agreed to in writing, software +dnl distributed under the License is distributed on an "AS IS" BASIS, +dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +dnl See the License for the specific language governing permissions and +dnl limitations under the License. +dnl + +dnl ------------------------------------------------------------------------- +dnl Author Pier Fumagalli +dnl Version $Id$ +dnl ------------------------------------------------------------------------- + +AC_DEFUN([AP_PROG_JAVAC_WORKS],[ + AC_CACHE_CHECK([wether the Java compiler ($JAVAC) works],ap_cv_prog_javac_works,[ + echo "public class Test {}" > Test.java + $JAVAC $JAVACFLAGS Test.java > /dev/null 2>&1 + if test $? -eq 0 + then + rm -f Test.java Test.class + ap_cv_prog_javac_works=yes + else + rm -f Test.java Test.class + AC_MSG_RESULT(no) + AC_MSG_ERROR([installation or configuration problem: javac cannot compile]) + fi + ]) +]) + +dnl Check for JAVA compilers. +AC_DEFUN([AP_PROG_JAVAC],[ + if test "$SABLEVM" != "NONE" + then + AC_PATH_PROG(JAVACSABLE,javac-sablevm,NONE,$JAVA_HOME/bin) + else + JAVACSABLE="NONE" + fi + if test "$JAVACSABLE" = "NONE" + then + XPATH="$JAVA_HOME/bin:$JAVA_HOME/Commands:$PATH" + AC_PATH_PROG(JAVAC,javac,NONE,$XPATH) + else + AC_PATH_PROG(JAVAC,javac-sablevm,NONE,$JAVA_HOME/bin) + fi + AC_MSG_RESULT([$JAVAC]) + if test "$JAVAC" = "NONE" + then + AC_MSG_ERROR([javac not found]) + fi + AP_PROG_JAVAC_WORKS() + AC_PROVIDE([$0]) + AC_SUBST(JAVAC) + AC_SUBST(JAVACFLAGS) +]) + +dnl Check for jar archivers. +AC_DEFUN([AP_PROG_JAR],[ + if test "$SABLEVM" != "NONE" + then + AC_PATH_PROG(JARSABLE,jar-sablevm,NONE,$JAVA_HOME/bin) + else + JARSABLE="NONE" + fi + if test "$JARSABLE" = "NONE" + then + XPATH="$JAVA_HOME/bin:$JAVA_HOME/Commands:$PATH" + AC_PATH_PROG(JAR,jar,NONE,$XPATH) + else + AC_PATH_PROG(JAR,jar-sablevm,NONE,$JAVA_HOME/bin) + fi + if test "$JAR" = "NONE" + then + AC_MSG_ERROR([jar not found]) + fi + AC_PROVIDE([$0]) + AC_SUBST(JAR) +]) + +AC_DEFUN([AP_JAVA],[ + AC_ARG_WITH(java,[ --with-java=DIR Specify the location of your JDK installation],[ + AC_MSG_CHECKING([JAVA_HOME]) + if test -d "$withval" + then + JAVA_HOME="$withval" + AC_MSG_RESULT([$JAVA_HOME]) + else + AC_MSG_RESULT([failed]) + AC_MSG_ERROR([$withval is not a directory]) + fi + AC_SUBST(JAVA_HOME) + ]) + if test x"$JAVA_HOME" = x + then + AC_MSG_ERROR([Java Home not defined. Rerun with --with-java=[...] parameter]) + fi +]) + +dnl check if the JVM in JAVA_HOME is sableVM +dnl $JAVA_HOME/bin/sablevm and /opt/java/lib/sablevm/bin are tested. +AC_DEFUN([AP_SABLEVM],[ + if test x"$JAVA_HOME" != x + then + AC_PATH_PROG(SABLEVM,sablevm,NONE,$JAVA_HOME/bin) + if test "$SABLEVM" = "NONE" + then + dnl java may be SableVM. + if $JAVA_HOME/bin/java -version 2> /dev/null | grep SableVM > /dev/null + then + SABLEVM=$JAVA_HOME/bin/java + fi + fi + if test "$SABLEVM" != "NONE" + then + AC_MSG_RESULT([Using sableVM: $SABLEVM]) + CFLAGS="$CFLAGS -DHAVE_SABLEVM" + fi + fi +]) + +dnl check if the JVM in JAVA_HOME is kaffe +dnl $JAVA_HOME/bin/kaffe is tested. +AC_DEFUN([AP_KAFFE],[ + if test x"$JAVA_HOME" != x + then + AC_PATH_PROG(KAFFEVM,kaffe,NONE,$JAVA_HOME/bin) + if test "$KAFFEVM" != "NONE" + then + AC_MSG_RESULT([Using kaffe: $KAFFEVM]) + CFLAGS="$CFLAGS -DHAVE_KAFFEVM" + LDFLAGS="$LDFLAGS -Wl,-rpath $JAVA_HOME/jre/lib/$HOST_CPU -L $JAVA_HOME/jre/lib/$HOST_CPU -lkaffevm" + fi + fi +]) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apsupport.m4 b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apsupport.m4 new file mode 100644 index 0000000000..0c8b262dcb --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apsupport.m4 @@ -0,0 +1,168 @@ +dnl +dnl Licensed to the Apache Software Foundation (ASF) under one or more +dnl contributor license agreements. See the NOTICE file distributed with +dnl this work for additional information regarding copyright ownership. +dnl The ASF licenses this file to You under the Apache License, Version 2.0 +dnl (the "License"); you may not use this file except in compliance with +dnl the License. You may obtain a copy of the License at +dnl +dnl http://www.apache.org/licenses/LICENSE-2.0 +dnl +dnl Unless required by applicable law or agreed to in writing, software +dnl distributed under the License is distributed on an "AS IS" BASIS, +dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +dnl See the License for the specific language governing permissions and +dnl limitations under the License. +dnl + +dnl ------------------------------------------------------------------------- +dnl Author Pier Fumagalli +dnl Version $Id$ +dnl ------------------------------------------------------------------------- + +AC_DEFUN([AP_SUPPORTED_HOST],[ + AC_MSG_CHECKING([C flags dependant on host system type]) + + case $host_os in + darwin*) + CFLAGS="$CFLAGS -DOS_DARWIN -DDSO_DYLD" + supported_os="darwin" + ;; + solaris*) + CFLAGS="$CFLAGS -DOS_SOLARIS -DDSO_DLFCN" + supported_os="solaris" + LIBS="$LIBS -ldl -lthread" + ;; + linux*) + CFLAGS="$CFLAGS -DOS_LINUX -DDSO_DLFCN" + supported_os="linux" + LIBS="$LIBS -ldl -lpthread" + ;; + cygwin) + CFLAGS="$CFLAGS -DOS_CYGWIN -DDSO_DLFCN -DNO_SETSID" + supported_os="win32" + ;; + sysv) + CFLAGS="$CFLAGS -DOS_SYSV -DDSO_DLFCN" + LIBS="$LIBS -ldl" + ;; + sysv4) + CFLAGS="$CFLAGS -DOS_SYSV -DDSO_DLFCN -Kthread" + LDFLAGS="-Kthread $LDFLAGS" + LIBS="$LIBS -ldl" + ;; + freebsd*) + CFLAGS="$CFLAGS -DOS_FREEBSD -DDSO_DLFCN -D_THREAD_SAFE -pthread" + LDFLAGS="-pthread $LDFLAGS" + supported_os="freebsd" + ;; + osf5*) + CFLAGS="$CFLAGS -pthread -DOS_TRU64 -DDSO_DLFCN -D_XOPEN_SOURCE_EXTENDED" + LDFLAGS="$LDFLAGS -pthread" + ;; + hpux11*) + CFLAGS="$CFLAGS -pthread -DOS_HPUX -DDSO_DLFCN" + LDFLAGS="$LDFLAGS -pthread" + LIBS="$LIBS -lpthread" + ;; + *) + AC_MSG_RESULT([failed]) + AC_MSG_ERROR([Unsupported operating system "$host_os"]);; + esac + + case $host_cpu in + powerpc*) + CFLAGS="$CFLAGS -DCPU=\\\"$host_cpu\\\"" + HOST_CPU=$host_cpu;; + sparc*) + CFLAGS="$CFLAGS -DCPU=\\\"$host_cpu\\\"" + HOST_CPU=$host_cpu;; + i?86) + CFLAGS="$CFLAGS -DCPU=\\\"i386\\\"" + HOST_CPU=i386;; + x86_64) + CFLAGS="$CFLAGS -DCPU=\\\"amd64\\\"" + HOST_CPU=amd64;; + bs2000) + CFLAGS="$CFLAGS -DCPU=\\\"osd\\\" -DCHARSET_EBCDIC -DOSD_POSIX" + supported_os="osd" + LDFLAGS="-Kno_link_stdlibs -B llm4" + LIBS="$LIBS -lBLSLIB" + LDCMD="/opt/C/bin/cc" + HOST_CPU=osd;; + mips) + CFLAGS="$CFLAGS -DCPU=\\\"mips\\\"" + supported_os="mips" + HOST_CPU=mips;; + alpha*) + CFLAGS="$CFLAGS -DCPU=\\\"alpha\\\"" + supported_os="alpha" + HOST_CPU=alpha;; + hppa2.0w) + CFLAGS="$CFLAGS -DCPU=\\\"PA_RISC2.0W\\\"" + supported_os="hp-ux" + HOST_CPU=PA_RISC2.0W;; + hppa2.0) + CFLAGS="$CFLAGS -DCPU=\\\"PA_RISC2.0\\\"" + supported_os="hp-ux" + HOST_CPU=PA_RISC2.0;; + mipsel) + CFLAGS="$CFLAGS -DCPU=\\\"mipsel\\\"" + supported_os="mipsel" + HOST_CPU=mipsel;; + ia64) + CFLAGS="$CFLAGS -DCPU=\\\"ia64\\\"" + supported_os="ia64" + HOST_CPU=ia64;; + s390) + CFLAGS="$CFLAGS -DCPU=\\\"s390\\\"" + supported_os="s390" + HOST_CPU=s390;; + arm*) + CFLAGS="$CFLAGS -DCPU=\\\"arm\\\"" + supported_os="arm" + HOST_CPU=arm;; + *) + AC_MSG_RESULT([failed]) + AC_MSG_ERROR([Unsupported CPU architecture "$host_cpu"]);; + esac + + AC_MSG_RESULT([ok]) + AC_SUBST(CFLAGS) + AC_SUBST(LDFLAGS) +]) + +AC_DEFUN([AP_JVM_LIBDIR],[ + AC_MSG_CHECKING([where on earth this jvm library is..]) + javabasedir=$JAVA_HOME + case $host_os in + cygwin* | mingw* | pw23* ) + lib_jvm_dir=`find $javabasedir -follow \( \ + \( -name client -type d -prune \) -o \ + \( -name "jvm.dll" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` + ;; + aix*) + lib_jvm_dir=`find $javabasedir \( \ + \( -name client -type d -prune \) -o \ + \( -name "libjvm.*" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` + if test -z "$lib_jvm_dir"; then + lib_jvm_dir=`find $javabasedir \( \ + \( -name client -type d -prune \) -o \ + \( -name "libkaffevm.*" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` + fi + ;; + *) + lib_jvm_dir=`find $javabasedir -follow \( \ + \( -name client -type d -prune \) -o \ + \( -name "libjvm.*" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` + if test -z "$lib_jvm_dir"; then + lib_jvm_dir=`find $javabasedir -follow \( \ + \( -name client -type d -prune \) -o \ + \( -name "libkaffevm.*" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` + fi + ;; + esac + LIB_JVM_DIR=$lib_jvm_dir + AC_MSG_RESULT([ohh u there ... $LIB_JVM_DIR]) + AC_SUBST(LIB_JVM_DIR) +]) diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml index 27161004a3..0e2684b1ea 100644 --- a/hadoop-hdfs-project/pom.xml +++ b/hadoop-hdfs-project/pom.xml @@ -34,6 +34,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> hadoop-hdfs hadoop-hdfs-httpfs hadoop-hdfs/src/contrib/bkjournal + hadoop-hdfs/src/contrib/fuse-dfs diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index f865b2dc59..fb3c97b521 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -47,37 +47,47 @@ - org.apache.maven.plugins - maven-antrun-plugin + org.codehaus.mojo + make-maven-plugin - make + compile compile - run - - - - - - - - - - - - + + autoreconf + configure + make-install + - native_tests + test test - - - - - - + + test + + + + ${project.build.directory}/native/container-executor + + -i + + + + + + CFLAGS + -DHADOOP_CONF_DIR=${container-executor.conf.dir} ${container-executor.additional_cflags} + + + ${project.build.directory}/native/container-executor + /usr/local + + + ${project.build.directory}/native/target + + @@ -162,6 +172,14 @@ run + + + + + + + + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt deleted file mode 100644 index e639507d26..0000000000 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cmake_minimum_required(VERSION 2.6 FATAL_ERROR) - -set(CMAKE_BUILD_TYPE, Release) - -if (JVM_ARCH_DATA_MODEL EQUAL 32) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") - set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32") -endif (JVM_ARCH_DATA_MODEL EQUAL 32) - -function(output_directory TGT DIR) - SET_TARGET_PROPERTIES(${TGT} PROPERTIES - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") - SET_TARGET_PROPERTIES(${TGT} PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") - SET_TARGET_PROPERTIES(${TGT} PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") -endfunction(output_directory TGT DIR) - -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2 -D_GNU_SOURCE") -# note: can't enable -D_LARGEFILE: see MAPREDUCE-4258 -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT") - -include_directories( - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_BINARY_DIR} - main/native/container-executor - main/native/container-executor/impl -) -CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h) - -add_library(container - main/native/container-executor/impl/configuration.c - main/native/container-executor/impl/container-executor.c -) - -add_executable(container-executor - main/native/container-executor/impl/main.c -) -target_link_libraries(container-executor - container -) -output_directory(container-executor target/usr/local/bin) - -add_executable(test-container-executor - main/native/container-executor/test/test-container-executor.c -) -target_link_libraries(test-container-executor - container -) -output_directory(test-container-executor target/usr/local/bin) diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake deleted file mode 100644 index 1fff36131f..0000000000 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef CONFIG_H -#define CONFIG_H - -#cmakedefine HADOOP_CONF_DIR "@HADOOP_CONF_DIR@" - -#endif diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.autom4te.cfg b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.autom4te.cfg new file mode 100644 index 0000000000..d21d1c9877 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.autom4te.cfg @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# autom4te configuration for hadoop utils library +# + +begin-language: "Autoheader-preselections" +args: --no-cache +end-language: "Autoheader-preselections" + +begin-language: "Automake-preselections" +args: --no-cache +end-language: "Automake-preselections" + +begin-language: "Autoreconf-preselections" +args: --no-cache +end-language: "Autoreconf-preselections" + +begin-language: "Autoconf-without-aclocal-m4" +args: --no-cache +end-language: "Autoconf-without-aclocal-m4" + +begin-language: "Autoconf" +args: --no-cache +end-language: "Autoconf" + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.deps/container-executor.Po b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.deps/container-executor.Po new file mode 100644 index 0000000000..9ce06a81ea --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.deps/container-executor.Po @@ -0,0 +1 @@ +# dummy diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/Makefile.am b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/Makefile.am new file mode 100644 index 0000000000..4938bb2f53 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/Makefile.am @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +AM_CFLAGS=-I$(srcdir)/impl -Wall -g -Werror + +# Define the programs that need to be built +bin_PROGRAMS = container-executor +check_PROGRAMS = test-container-executor + +TESTS = test-container-executor + +# Define the sources for the common files +common_SOURCES = impl/configuration.c impl/container-executor.c + +# Define the sources for the real executable +container_executor_SOURCES = $(common_SOURCES) impl/main.c + +# Define the sources for the test executable +test_container_executor_SOURCES = $(common_SOURCES) test/test-container-executor.c diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/configure.ac b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/configure.ac new file mode 100644 index 0000000000..db8af88cf1 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/configure.ac @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# -*- Autoconf -*- +# Process this file with autoconf to produce a configure script. + +AC_PREREQ(2.59) +AC_INIT(linux-container-executor, 1.0.0, mapreduce-dev@hadoop.apache.org) +AC_GNU_SOURCE +#AC_SYS_LARGEFILE + +AM_INIT_AUTOMAKE([subdir-objects foreign no-dist]) + +AC_CONFIG_SRCDIR([impl/container-executor.c]) +AC_CONFIG_FILES([Makefile]) + +AC_PREFIX_DEFAULT(`pwd`/../install) + +CHECK_INSTALL_CFLAG +HADOOP_UTILS_SETUP + +# Checks for programs. +AC_PROG_CC +AM_PROG_CC_C_O + +# Checks for libraries. + +# Checks for header files. +AC_LANG(C) +AC_CHECK_HEADERS([unistd.h]) + +# Checks for typedefs, structures, and compiler characteristics. +AC_HEADER_STDBOOL +AC_C_CONST +AC_TYPE_OFF_T +AC_TYPE_SIZE_T +AC_FUNC_STRERROR_R + +# Checks for library functions. +AC_CHECK_FUNCS([mkdir uname]) +AC_OUTPUT diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index d6ce5aa706..cd8caabe33 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -16,7 +16,6 @@ * limitations under the License. */ -#include "config.h" #include "configuration.h" #include "container-executor.h" @@ -30,6 +29,8 @@ #include #include +#define _STRINGIFY(X) #X +#define STRINGIFY(X) _STRINGIFY(X) #define CONF_FILENAME "container-executor.cfg" // When building as part of a Maven build this value gets defined by using @@ -100,7 +101,7 @@ int main(int argc, char **argv) { char *executable_file = get_executable(); - char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME; + char *orig_conf_file = STRINGIFY(HADOOP_CONF_DIR) "/" CONF_FILENAME; char *conf_file = resolve_config_path(orig_conf_file, argv[0]); char *local_dirs, *log_dirs; From 9b1e5279f843be99fc89c6d13e135594f5032aa1 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Thu, 7 Jun 2012 18:41:24 +0000 Subject: [PATCH 07/91] HDFS-3485. DataTransferThrottler will over-throttle when currentTimeMillis jumps. Contributed by Andy Isaacson. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1347751 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../apache/hadoop/hdfs/server/common/Util.java | 17 ++++++++++++++++- .../hadoop/hdfs/util/DataTransferThrottler.java | 12 +++++++----- .../server/datanode/TestBlockReplacement.java | 12 +++++++++--- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 03d8c8b679..f0bc858f8f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -313,6 +313,9 @@ Branch-2 ( Unreleased changes ) HDFS-3492. Fix some misuses of InputStream#skip (Colin Patrick McCabe via todd) + HDFS-3485. DataTransferThrottler will over-throttle when currentTimeMillis + jumps (Andy Isaacson via todd) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java index 1f4e974166..7c8c301180 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java @@ -34,13 +34,28 @@ public final class Util { private final static Log LOG = LogFactory.getLog(Util.class.getName()); /** - * Current system time. + * Current system time. Do not use this to calculate a duration or interval + * to sleep, because it will be broken by settimeofday. Instead, use + * monotonicNow. * @return current time in msec. */ public static long now() { return System.currentTimeMillis(); } + /** + * Current time from some arbitrary time base in the past, counting in + * milliseconds, and not affected by settimeofday or similar system clock + * changes. This is appropriate to use when computing how much longer to + * wait for an interval to expire. + * @return a monotonic clock that counts in milliseconds. + */ + public static long monotonicNow() { + final long NANOSECONDS_PER_MILLISECOND = 1000000; + + return System.nanoTime() / NANOSECONDS_PER_MILLISECOND; + } + /** * Interprets the passed string as a URI. In case of error it * assumes the specified string is a file. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/DataTransferThrottler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/DataTransferThrottler.java index 8ef570181a..d452205cc6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/DataTransferThrottler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/DataTransferThrottler.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.util; +import static org.apache.hadoop.hdfs.server.common.Util.monotonicNow; + /** * a class to throttle the data transfers. * This class is thread safe. It can be shared by multiple threads. @@ -26,9 +28,9 @@ public class DataTransferThrottler { private long period; // period over which bw is imposed private long periodExtension; // Max period over which bw accumulates. - private long bytesPerPeriod; // total number of bytes can be sent in each period - private long curPeriodStart; // current period starting time - private long curReserve; // remaining bytes can be sent in the period + private long bytesPerPeriod; // total number of bytes can be sent in each period + private long curPeriodStart; // current period starting time + private long curReserve; // remaining bytes can be sent in the period private long bytesAlreadyUsed; /** Constructor @@ -45,7 +47,7 @@ public DataTransferThrottler(long bandwidthPerSec) { * @param bandwidthPerSec bandwidth allowed in bytes per second. */ public DataTransferThrottler(long period, long bandwidthPerSec) { - this.curPeriodStart = System.currentTimeMillis(); + this.curPeriodStart = monotonicNow(); this.period = period; this.curReserve = this.bytesPerPeriod = bandwidthPerSec*period/1000; this.periodExtension = period*3; @@ -87,7 +89,7 @@ public synchronized void throttle(long numOfBytes) { bytesAlreadyUsed += numOfBytes; while (curReserve <= 0) { - long now = System.currentTimeMillis(); + long now = monotonicNow(); long curPeriodEnd = curPeriodStart + period; if ( now < curPeriodEnd ) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java index 78d20ad655..8584cc8266 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java @@ -28,8 +28,6 @@ import java.util.Random; import java.util.concurrent.TimeoutException; -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -52,14 +50,21 @@ import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.net.NetUtils; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + /** * This class tests if block replacement request to data nodes work correctly. */ -public class TestBlockReplacement extends TestCase { +public class TestBlockReplacement { private static final Log LOG = LogFactory.getLog( "org.apache.hadoop.hdfs.TestBlockReplacement"); MiniDFSCluster cluster; + @Test public void testThrottler() throws IOException { Configuration conf = new HdfsConfiguration(); FileSystem.setDefaultUri(conf, "hdfs://localhost:0"); @@ -83,6 +88,7 @@ public void testThrottler() throws IOException { assertTrue(totalBytes*1000/(end-start)<=bandwidthPerSec); } + @Test public void testBlockReplacement() throws IOException, TimeoutException { final Configuration CONF = new HdfsConfiguration(); final String[] INITIAL_RACKS = {"/RACK0", "/RACK1", "/RACK2"}; From 0b658968a062a1f43e2ab01a11678dc8e9928ec6 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Thu, 7 Jun 2012 21:06:13 +0000 Subject: [PATCH 08/91] Revert HDFS-3492 from r1347192: patch broke TestShortCircuitLocalRead git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1347796 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/lib/wsrs/InputStreamEntity.java | 5 ++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 -- .../apache/hadoop/hdfs/BlockReaderLocal.java | 34 +++++++++++++++---- .../hdfs/server/namenode/FSEditLogOp.java | 7 ++-- .../hdfs/TestShortCircuitLocalRead.java | 7 ++-- .../server/datanode/SimulatedFSDataset.java | 3 +- 6 files changed, 38 insertions(+), 21 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/InputStreamEntity.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/InputStreamEntity.java index 21c25bd4c5..e5361a80ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/InputStreamEntity.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/InputStreamEntity.java @@ -42,7 +42,10 @@ public InputStreamEntity(InputStream is) { @Override public void write(OutputStream os) throws IOException { - IOUtils.skipFully(is, offset); + long skipped = is.skip(offset); + if (skipped < offset) { + throw new IOException("Requested offset beyond stream size"); + } if (len == -1) { IOUtils.copyBytes(is, os, 4096, true); } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f0bc858f8f..2fad18ecc7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -310,9 +310,6 @@ Branch-2 ( Unreleased changes ) HDFS-3505. DirectoryScanner does not join all threads in shutdown. (Colin Patrick McCabe via eli) - HDFS-3492. Fix some misuses of InputStream#skip (Colin Patrick McCabe - via todd) - HDFS-3485. DataTransferThrottler will over-throttle when currentTimeMillis jumps (Andy Isaacson via todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java index a51b710e29..cd85ebae05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java @@ -39,7 +39,6 @@ import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader; import org.apache.hadoop.hdfs.util.DirectBufferPool; import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.DataChecksum; @@ -316,10 +315,23 @@ private BlockReaderLocal(Configuration conf, String hdfsfile, boolean success = false; try { // Skip both input streams to beginning of the chunk containing startOffset - IOUtils.skipFully(dataIn, firstChunkOffset); + long toSkip = firstChunkOffset; + while (toSkip > 0) { + long skipped = dataIn.skip(toSkip); + if (skipped == 0) { + throw new IOException("Couldn't initialize input stream"); + } + toSkip -= skipped; + } if (checksumIn != null) { long checkSumOffset = (firstChunkOffset / bytesPerChecksum) * checksumSize; - IOUtils.skipFully(dataIn, checkSumOffset); + while (checkSumOffset > 0) { + long skipped = checksumIn.skip(checkSumOffset); + if (skipped == 0) { + throw new IOException("Couldn't initialize checksum input stream"); + } + checkSumOffset -= skipped; + } } success = true; } finally { @@ -624,9 +636,17 @@ public synchronized long skip(long n) throws IOException { slowReadBuff.position(slowReadBuff.limit()); checksumBuff.position(checksumBuff.limit()); - IOUtils.skipFully(dataIn, toskip); - long checkSumOffset = (toskip / bytesPerChecksum) * checksumSize; - IOUtils.skipFully(checksumIn, checkSumOffset); + long dataSkipped = dataIn.skip(toskip); + if (dataSkipped != toskip) { + throw new IOException("skip error in data input stream"); + } + long checkSumOffset = (dataSkipped / bytesPerChecksum) * checksumSize; + if (checkSumOffset > 0) { + long skipped = checksumIn.skip(checkSumOffset); + if (skipped != checkSumOffset) { + throw new IOException("skip error in checksum input stream"); + } + } // read into the middle of the chunk if (skipBuf == null) { @@ -681,4 +701,4 @@ public Socket takeSocket() { public boolean hasSentStatusCode() { return false; } -} +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java index 2aa8f736c1..489f030e13 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java @@ -44,7 +44,6 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.ArrayWritable; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableFactories; @@ -2290,11 +2289,9 @@ public FSEditLogOp readOp(boolean skipBrokenEdits) throws IOException { // 0xff, we want to skip over that region, because there's nothing // interesting there. long numSkip = e.getNumAfterTerminator(); - try { - IOUtils.skipFully(in, numSkip); - } catch (IOException t) { + if (in.skip(numSkip) < numSkip) { FSImage.LOG.error("Failed to skip " + numSkip + " bytes of " + - "garbage after an OP_INVALID. Unexpected early EOF.", t); + "garbage after an OP_INVALID. Unexpected early EOF."); return null; } } catch (IOException e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java index 9c37b7a2fb..169756759e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.junit.Assert; @@ -95,7 +94,8 @@ static void checkFileContent(FileSystem fs, Path name, byte[] expected, // Now read using a different API. actual = new byte[expected.length-readOffset]; stm = fs.open(name); - IOUtils.skipFully(stm, readOffset); + long skipped = stm.skip(readOffset); + Assert.assertEquals(skipped, readOffset); //Read a small number of bytes first. int nread = stm.read(actual, 0, 3); nread += stm.read(actual, nread, 2); @@ -123,7 +123,8 @@ static void checkFileContentDirect(FileSystem fs, Path name, byte[] expected, ByteBuffer actual = ByteBuffer.allocate(expected.length - readOffset); - IOUtils.skipFully(stm, readOffset); + long skipped = stm.skip(readOffset); + Assert.assertEquals(skipped, readOffset); actual.limit(3); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index d4a7b3a257..e69b1c3021 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -47,7 +47,6 @@ import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.DiskChecker.DiskErrorException; @@ -687,7 +686,7 @@ synchronized InputStream getBlockInputStream(ExtendedBlock b public synchronized InputStream getBlockInputStream(ExtendedBlock b, long seekOffset) throws IOException { InputStream result = getBlockInputStream(b); - IOUtils.skipFully(result, seekOffset); + result.skip(seekOffset); return result; } From bb614783f155ed0fa1b6206f5669ba430444a0a9 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Thu, 7 Jun 2012 21:35:30 +0000 Subject: [PATCH 09/91] Move HDFS-3042 (automatic failover) to branch-2 section of CHANGES.txt git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1347809 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 70 +++++++++---------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 38 +++++----- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 477e336c1e..b6a3631ee3 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -12,9 +12,6 @@ Trunk (unreleased changes) HADOOP-8135. Add ByteBufferReadable interface to FSDataInputStream. (Henry Robinson via atm) - HDFS-3042. Automatic failover support for NameNode HA (todd) - (see dedicated section below for breakdown of subtasks) - IMPROVEMENTS HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution @@ -165,38 +162,6 @@ Trunk (unreleased changes) HADOOP-7761. Improve the performance of raw comparisons. (todd) - BREAKDOWN OF HDFS-3042 SUBTASKS - - HADOOP-8220. ZKFailoverController doesn't handle failure to become active - correctly (todd) - - HADOOP-8228. Auto HA: Refactor tests and add stress tests. (todd) - - HADOOP-8215. Security support for ZK Failover controller (todd) - - HADOOP-8245. Fix flakiness in TestZKFailoverController (todd) - - HADOOP-8257. TestZKFailoverControllerStress occasionally fails with Mockito - error (todd) - - HADOOP-8260. Replace ClientBaseWithFixes with our own modified copy of the - class (todd) - - HADOOP-8246. Auto-HA: automatically scope znode by nameservice ID (todd) - - HADOOP-8247. Add a config to enable auto-HA, which disables manual - FailoverController (todd) - - HADOOP-8306. ZKFC: improve error message when ZK is not running. (todd) - - HADOOP-8279. Allow manual failover to be invoked when auto-failover is - enabled. (todd) - - HADOOP-8276. Auto-HA: add config for java options to pass to zkfc daemon - (todd via eli) - - HADOOP-8405. ZKFC tests leak ZK instances. (todd) - Branch-2 ( Unreleased changes ) INCOMPATIBLE CHANGES @@ -206,6 +171,9 @@ Branch-2 ( Unreleased changes ) NEW FEATURES + HDFS-3042. Automatic failover support for NameNode HA (todd) + (see dedicated section below for breakdown of subtasks) + IMPROVEMENTS HADOOP-8340. SNAPSHOT build versions should compare as less than their eventual @@ -269,6 +237,38 @@ Branch-2 ( Unreleased changes ) HADOOP-8485. Don't hardcode "Apache Hadoop 0.23" in the docs. (eli) + BREAKDOWN OF HDFS-3042 SUBTASKS + + HADOOP-8220. ZKFailoverController doesn't handle failure to become active + correctly (todd) + + HADOOP-8228. Auto HA: Refactor tests and add stress tests. (todd) + + HADOOP-8215. Security support for ZK Failover controller (todd) + + HADOOP-8245. Fix flakiness in TestZKFailoverController (todd) + + HADOOP-8257. TestZKFailoverControllerStress occasionally fails with Mockito + error (todd) + + HADOOP-8260. Replace ClientBaseWithFixes with our own modified copy of the + class (todd) + + HADOOP-8246. Auto-HA: automatically scope znode by nameservice ID (todd) + + HADOOP-8247. Add a config to enable auto-HA, which disables manual + FailoverController (todd) + + HADOOP-8306. ZKFC: improve error message when ZK is not running. (todd) + + HADOOP-8279. Allow manual failover to be invoked when auto-failover is + enabled. (todd) + + HADOOP-8276. Auto-HA: add config for java options to pass to zkfc daemon + (todd via eli) + + HADOOP-8405. ZKFC tests leak ZK instances. (todd) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2fad18ecc7..504faffdf7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -15,9 +15,6 @@ Trunk (unreleased changes) HDFS-744. Support hsync in HDFS. (Lars Hofhansl via szetszwo) - HDFS-3042. Automatic failover support for NameNode HA (todd) - (see dedicated section below for subtask breakdown) - IMPROVEMENTS HDFS-1620. Rename HdfsConstants -> HdfsServerConstants, FSConstants -> @@ -162,28 +159,15 @@ Trunk (unreleased changes) HDFS-3462. TestDFSClientRetries.busyTest() should restore default xceiver count in the config. (Madhukara Phatak via harsh) - BREAKDOWN OF HDFS-3042 SUBTASKS - - HDFS-2185. HDFS portion of ZK-based FailoverController (todd) - - HDFS-3200. Scope all ZKFC configurations by nameservice (todd) - - HDFS-3223. add zkfc to hadoop-daemon.sh script (todd) - - HDFS-3261. TestHASafeMode fails on HDFS-3042 branch (todd) - - HDFS-3159. Document NN auto-failover setup and configuration (todd) - - HDFS-3412. Fix findbugs warnings in auto-HA branch (todd) - - HDFS-3432. TestDFSZKFailoverController tries to fail over too early (todd) - Branch-2 ( Unreleased changes ) INCOMPATIBLE CHANGES NEW FEATURES + HDFS-3042. Automatic failover support for NameNode HA (todd) + (see dedicated section below for breakdown of subtasks) + IMPROVEMENTS HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG @@ -313,6 +297,22 @@ Branch-2 ( Unreleased changes ) HDFS-3485. DataTransferThrottler will over-throttle when currentTimeMillis jumps (Andy Isaacson via todd) + BREAKDOWN OF HDFS-3042 SUBTASKS + + HDFS-2185. HDFS portion of ZK-based FailoverController (todd) + + HDFS-3200. Scope all ZKFC configurations by nameservice (todd) + + HDFS-3223. add zkfc to hadoop-daemon.sh script (todd) + + HDFS-3261. TestHASafeMode fails on HDFS-3042 branch (todd) + + HDFS-3159. Document NN auto-failover setup and configuration (todd) + + HDFS-3412. Fix findbugs warnings in auto-HA branch (todd) + + HDFS-3432. TestDFSZKFailoverController tries to fail over too early (todd) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES From a98e01a5b4a62e3d55cc23e1587b8445b305a6f7 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Fri, 8 Jun 2012 00:44:39 +0000 Subject: [PATCH 10/91] HADOOP-8488. test-patch.sh gives +1 even if the native build fails. Contributed by Colin Patrick McCabe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1347846 13f79535-47bb-0310-9956-ffa450edef68 --- dev-support/test-patch.sh | 4 ++-- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index fb75b90be5..9d4d6194fa 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -418,8 +418,8 @@ checkJavacWarnings () { echo "======================================================================" echo "" echo "" - echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1" - $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1 + echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Pnative -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1" + $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Pnative -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1 if [[ $? != 0 ]] ; then JIRA_COMMENT="$JIRA_COMMENT diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b6a3631ee3..37148039d2 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -269,6 +269,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8405. ZKFC tests leak ZK instances. (todd) + HADOOP-8488. test-patch.sh gives +1 even if the native build fails. + (Colin Patrick McCabe via eli) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES From a6cc4e761b75627d0337ac461d19fd46f7c81362 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Fri, 8 Jun 2012 02:29:40 +0000 Subject: [PATCH 11/91] HADOOP-8469. Make NetworkTopology class pluggable. Contributed by Junping Du git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1347867 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../fs/CommonConfigurationKeysPublic.java | 4 +- .../apache/hadoop/net/NetworkTopology.java | 171 +++++++++++++++--- .../hadoop/hdfs/protocol/DatanodeInfo.java | 4 +- .../blockmanagement/DatanodeManager.java | 24 ++- 5 files changed, 171 insertions(+), 35 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 37148039d2..4ac177755a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -12,6 +12,9 @@ Trunk (unreleased changes) HADOOP-8135. Add ByteBufferReadable interface to FSDataInputStream. (Henry Robinson via atm) + HADOOP-8469. Make NetworkTopology class pluggable. (Junping Du via + szetszwo) + IMPROVEMENTS HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 67f3bc594c..c367294399 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -63,7 +63,9 @@ public class CommonConfigurationKeysPublic { /** See core-default.xml */ public static final String NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY = "net.topology.node.switch.mapping.impl"; - + /** See core-default.xml */ + public static final String NET_TOPOLOGY_IMPL_KEY = + "net.topology.impl"; /** See core-default.xml */ public static final String NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY = "net.topology.table.file.name"; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java index da8fab2956..892ba07359 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Random; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -55,8 +56,8 @@ public InvalidTopologyException(String msg) { /** InnerNode represents a switch/router of a data center or rack. * Different from a leaf node, it has non-null children. */ - private class InnerNode extends NodeBase { - private ArrayList children=new ArrayList(); + static class InnerNode extends NodeBase { + protected List children=new ArrayList(); private int numOfLeaves; /** Construct an InnerNode from a path-like string */ @@ -76,7 +77,7 @@ private class InnerNode extends NodeBase { } /** @return its children */ - Collection getChildren() {return children;} + List getChildren() {return children;} /** @return the number of children this node has */ int getNumOfChildren() { @@ -182,7 +183,23 @@ boolean add(Node n) { } } } - + + /** + * Creates a parent node to be added to the list of children. + * Creates a node using the InnerNode four argument constructor specifying + * the name, location, parent, and level of this node. + * + *

To be overridden in subclasses for specific InnerNode implementations, + * as alternative to overriding the full {@link #add(Node)} method. + * + * @param parentName The name of the parent node + * @return A new inner node + * @see InnerNode#InnerNode(String, String, InnerNode, int) + */ + protected InnerNode createParentNode(String parentName) { + return new InnerNode(parentName, getPath(this), this, this.getLevel()+1); + } + /** Remove node n from the subtree of this node * @param n node to be deleted * @return true if the node is deleted; false otherwise @@ -263,7 +280,7 @@ private Node getLoc(String loc) { * @param excludedNode an excluded node (can be null) * @return */ - private Node getLeaf(int leafIndex, Node excludedNode) { + Node getLeaf(int leafIndex, Node excludedNode) { int count=0; // check if the excluded node a leaf boolean isLeaf = @@ -308,7 +325,21 @@ private Node getLeaf(int leafIndex, Node excludedNode) { return null; } } - + + /** + * Determine if children a leaves, default implementation calls {@link #isRack()} + *

To be overridden in subclasses for specific InnerNode implementations, + * as alternative to overriding the full {@link #getLeaf(int, Node)} method. + * + * @return true if children are leaves, false otherwise + */ + protected boolean areChildrenLeaves() { + return isRack(); + } + + /** + * Get number of leaves. + */ int getNumOfLeaves() { return numOfLeaves; } @@ -317,18 +348,18 @@ int getNumOfLeaves() { /** * the root cluster map */ - InnerNode clusterMap = new InnerNode(InnerNode.ROOT); + InnerNode clusterMap; /** Depth of all leaf nodes */ private int depthOfAllLeaves = -1; /** rack counter */ - private int numOfRacks = 0; + protected int numOfRacks = 0; /** the lock used to manage access */ - private ReadWriteLock netlock; - + protected ReadWriteLock netlock = new ReentrantReadWriteLock(); + public NetworkTopology() { - netlock = new ReentrantReadWriteLock(); + clusterMap = new InnerNode(InnerNode.ROOT); } - + /** Add a leaf node * Update node counter & rack counter if necessary * @param node node to be added; can be null @@ -344,7 +375,7 @@ public void add(Node node) { } netlock.writeLock().lock(); try { - Node rack = getNode(node.getNetworkLocation()); + Node rack = getNodeForNetworkLocation(node); if (rack != null && !(rack instanceof InnerNode)) { throw new IllegalArgumentException("Unexpected data node " + node.toString() @@ -376,7 +407,26 @@ public void add(Node node) { netlock.writeLock().unlock(); } } - + + /** + * Return a reference to the node given its string representation. + * Default implementation delegates to {@link #getNode(String)}. + * + *

To be overridden in subclasses for specific NetworkTopology + * implementations, as alternative to overriding the full {@link #add(Node)} + * method. + * + * @param node The string representation of this node's network location is + * used to retrieve a Node object. + * @return a reference to the node; null if the node is not in the tree + * + * @see #add(Node) + * @see #getNode(String) + */ + protected Node getNodeForNetworkLocation(Node node) { + return getNode(node.getNetworkLocation()); + } + /** Remove a node * Update node counter and rack counter if necessary * @param node node to be removed; can be null @@ -403,7 +453,7 @@ public void remove(Node node) { netlock.writeLock().unlock(); } } - + /** Check if the tree contains node node * * @param node a node @@ -443,7 +493,21 @@ public Node getNode(String loc) { netlock.readLock().unlock(); } } - + + /** Given a string representation of a rack for a specific network + * location + * + * To be overridden in subclasses for specific NetworkTopology + * implementations, as alternative to overriding the full + * {@link #getRack(String)} method. + * @param loc + * a path-like string representation of a network location + * @return a rack string + */ + public String getRack(String loc) { + return loc; + } + /** @return the total number of racks */ public int getNumOfRacks() { netlock.readLock().lock(); @@ -453,7 +517,7 @@ public int getNumOfRacks() { netlock.readLock().unlock(); } } - + /** @return the total number of leaf nodes */ public int getNumOfLeaves() { netlock.readLock().lock(); @@ -463,7 +527,7 @@ public int getNumOfLeaves() { netlock.readLock().unlock(); } } - + /** Return the distance between two nodes * It is assumed that the distance from one node to its parent is 1 * The distance between two nodes is calculated by summing up their distances @@ -509,8 +573,8 @@ public int getDistance(Node node1, Node node2) { return Integer.MAX_VALUE; } return dis+2; - } - + } + /** Check if two nodes are on the same rack * @param node1 one node (can be null) * @param node2 another node (can be null) @@ -525,13 +589,44 @@ public boolean isOnSameRack( Node node1, Node node2) { netlock.readLock().lock(); try { - return node1.getParent()==node2.getParent(); + return isSameParents(node1, node2); } finally { netlock.readLock().unlock(); } } - - final private static Random r = new Random(); + + /** + * Check if network topology is aware of NodeGroup + */ + public boolean isNodeGroupAware() { + return false; + } + + /** + * Return false directly as not aware of NodeGroup, to be override in sub-class + */ + public boolean isOnSameNodeGroup(Node node1, Node node2) { + return false; + } + + /** + * Compare the parents of each node for equality + * + *

To be overridden in subclasses for specific NetworkTopology + * implementations, as alternative to overriding the full + * {@link #isOnSameRack(Node, Node)} method. + * + * @param node1 the first node to compare + * @param node2 the second node to compare + * @return true if their parents are equal, false otherwise + * + * @see #isOnSameRack(Node, Node) + */ + protected boolean isSameParents(Node node1, Node node2) { + return node1.getParent()==node2.getParent(); + } + + final protected static Random r = new Random(); /** randomly choose one node from scope * if scope starts with ~, choose one from the all nodes except for the * ones in scope; otherwise, choose one from scope @@ -550,7 +645,7 @@ public Node chooseRandom(String scope) { netlock.readLock().unlock(); } } - + private Node chooseRandom(String scope, String excludedScope){ if (excludedScope != null) { if (scope.startsWith(excludedScope)) { @@ -579,7 +674,25 @@ private Node chooseRandom(String scope, String excludedScope){ int leaveIndex = r.nextInt(numOfDatanodes); return innerNode.getLeaf(leaveIndex, node); } - + + /** return leaves in scope + * @param scope a path string + * @return leaves nodes under specific scope + */ + public List getLeaves(String scope) { + Node node = getNode(scope); + List leafNodes = new ArrayList(); + if (!(node instanceof InnerNode)) { + leafNodes.add(node); + } else { + InnerNode innerNode = (InnerNode) node; + for (int i=0;iscope but not in excludedNodes * if scope starts with ~, return the number of nodes that are not * in scope and excludedNodes; @@ -619,7 +732,7 @@ public int countNumOfAvailableNodes(String scope, netlock.readLock().unlock(); } } - + /** convert a network tree to a string */ public String toString() { // print the number of racks @@ -640,13 +753,12 @@ public String toString() { return tree.toString(); } - /* swap two array items */ - static private void swap(Node[] nodes, int i, int j) { + /** swap two array items */ + static protected void swap(Node[] nodes, int i, int j) { Node tempNode; tempNode = nodes[j]; nodes[j] = nodes[i]; nodes[i] = tempNode; - } /** Sort nodes array by their distances to reader @@ -697,4 +809,5 @@ public void pseudoSortByDistance( Node reader, Node[] nodes ) { swap(nodes, 0, r.nextInt(nodes.length)); } } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java index db7eaed643..0fd1e7d51d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java @@ -199,10 +199,10 @@ public void setXceiverCount(int xceiverCount) { this.xceiverCount = xceiverCount; } - /** rack name */ + /** network location */ public synchronized String getNetworkLocation() {return location;} - /** Sets the rack name */ + /** Sets the network location */ public synchronized void setNetworkLocation(String location) { this.location = NodeBase.normalize(location); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 2aee0eb5ee..c58b857ebf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -38,6 +38,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -66,6 +67,8 @@ import org.apache.hadoop.net.CachedDNSToSwitchMapping; import org.apache.hadoop.net.DNSToSwitchMapping; import org.apache.hadoop.net.NetworkTopology; +import org.apache.hadoop.net.Node; +import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.net.ScriptBasedMapping; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.HostsFileReader; @@ -108,7 +111,7 @@ public class DatanodeManager { = new TreeMap(); /** Cluster network topology */ - private final NetworkTopology networktopology = new NetworkTopology(); + private final NetworkTopology networktopology; /** Host names to datanode descriptors mapping. */ private final Host2NodesMap host2DatanodeMap = new Host2NodesMap(); @@ -134,6 +137,12 @@ public class DatanodeManager { ) throws IOException { this.namesystem = namesystem; this.blockManager = blockManager; + + Class networkTopologyClass = + conf.getClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY, + NetworkTopology.class, NetworkTopology.class); + networktopology = (NetworkTopology) ReflectionUtils.newInstance( + networkTopologyClass, conf); this.heartbeatManager = new HeartbeatManager(namesystem, blockManager, conf); @@ -206,13 +215,22 @@ public DatanodeStatistics getDatanodeStatistics() { public void sortLocatedBlocks(final String targethost, final List locatedblocks) { //sort the blocks - final DatanodeDescriptor client = getDatanodeByHost(targethost); + // As it is possible for the separation of node manager and datanode, + // here we should get node but not datanode only . + Node client = getDatanodeByHost(targethost); + if (client == null) { + List hosts = new ArrayList (1); + hosts.add(targethost); + String rName = dnsToSwitchMapping.resolve(hosts).get(0); + if (rName != null) + client = new NodeBase(rName + NodeBase.PATH_SEPARATOR_STR + targethost); + } for (LocatedBlock b : locatedblocks) { networktopology.pseudoSortByDistance(client, b.getLocations()); // Move decommissioned datanodes to the bottom Arrays.sort(b.getLocations(), DFSUtil.DECOM_COMPARATOR); - } + } } CyclicIteration getDatanodeCyclicIteration( From 20e8b1db2a8253d990d70c7efb0c24a1d5b6a4d7 Mon Sep 17 00:00:00 2001 From: Aaron Twining Myers Date: Fri, 8 Jun 2012 06:38:14 +0000 Subject: [PATCH 12/91] HDFS-3040. TestMulitipleNNDataBlockScanner is misspelled. Contributed by Madhukara Phatak. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1347891 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ ...aBlockScanner.java => TestMultipleNNDataBlockScanner.java} | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) rename hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/{TestMulitipleNNDataBlockScanner.java => TestMultipleNNDataBlockScanner.java} (98%) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 504faffdf7..ed1bdd39c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -87,6 +87,9 @@ Trunk (unreleased changes) HDFS-3476. Correct the default used in TestDFSClientRetries.busyTest() after HDFS-3462 (harsh) + HDFS-3040. TestMulitipleNNDataBlockScanner is misspelled. (Madhukara Phatak + via atm) + OPTIMIZATIONS HDFS-2834. Add a ByteBuffer-based read API to DFSInputStream. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMulitipleNNDataBlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMultipleNNDataBlockScanner.java similarity index 98% rename from hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMulitipleNNDataBlockScanner.java rename to hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMultipleNNDataBlockScanner.java index 14a0da3804..9b52252f36 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMulitipleNNDataBlockScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMultipleNNDataBlockScanner.java @@ -34,9 +34,9 @@ import org.junit.Test; -public class TestMulitipleNNDataBlockScanner { +public class TestMultipleNNDataBlockScanner { private static final Log LOG = - LogFactory.getLog(TestMulitipleNNDataBlockScanner.class); + LogFactory.getLog(TestMultipleNNDataBlockScanner.class); Configuration conf; MiniDFSCluster cluster = null; String bpids[] = new String[3]; From f7e63759bd833a1ea09541c3a52745a3e5792f42 Mon Sep 17 00:00:00 2001 From: Aaron Twining Myers Date: Fri, 8 Jun 2012 06:57:52 +0000 Subject: [PATCH 13/91] HDFS-2914. HA: Standby should not enter safemode when resources are low. Contributed by Vinay. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1347895 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/namenode/FSNamesystem.java | 20 ++++++++++++++----- .../namenode/ha/TestFailureOfSharedDir.java | 12 +++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ed1bdd39c6..beb6389ac1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -300,6 +300,9 @@ Branch-2 ( Unreleased changes ) HDFS-3485. DataTransferThrottler will over-throttle when currentTimeMillis jumps (Andy Isaacson via todd) + HDFS-2914. HA: Standby should not enter safemode when resources are low. + (Vinay via atm) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index fbca355b25..386b5eb467 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -572,8 +572,6 @@ void startCommonServices(Configuration conf, HAContext haContext) throws IOExcep !safeMode.isPopulatingReplQueues(); setBlockTotal(); blockManager.activate(conf); - this.nnrmthread = new Daemon(new NameNodeResourceMonitor()); - nnrmthread.start(); } finally { writeUnlock(); } @@ -590,7 +588,6 @@ void stopCommonServices() { writeLock(); try { if (blockManager != null) blockManager.close(); - if (nnrmthread != null) nnrmthread.interrupt(); } finally { writeUnlock(); } @@ -644,6 +641,10 @@ void startActiveServices() throws IOException { } leaseManager.startMonitor(); startSecretManagerIfNecessary(); + + //ResourceMonitor required only at ActiveNN. See HDFS-2914 + this.nnrmthread = new Daemon(new NameNodeResourceMonitor()); + nnrmthread.start(); } finally { writeUnlock(); } @@ -666,6 +667,10 @@ void stopActiveServices() { if (leaseManager != null) { leaseManager.stopMonitor(); } + if (nnrmthread != null) { + ((NameNodeResourceMonitor) nnrmthread.getRunnable()).stopMonitor(); + nnrmthread.interrupt(); + } if (dir != null && dir.fsImage != null) { if (dir.fsImage.editLog != null) { dir.fsImage.editLog.close(); @@ -3193,10 +3198,11 @@ void checkAvailableResources() { * acceptable levels, this daemon will cause the NN to exit safe mode. */ class NameNodeResourceMonitor implements Runnable { + boolean shouldNNRmRun = true; @Override public void run () { try { - while (fsRunning) { + while (fsRunning && shouldNNRmRun) { checkAvailableResources(); if(!nameNodeHasResourcesAvailable()) { String lowResourcesMsg = "NameNode low on available disk space. "; @@ -3217,7 +3223,11 @@ public void run () { FSNamesystem.LOG.error("Exception in NameNodeResourceMonitor: ", e); } } - } + + public void stopMonitor() { + shouldNNRmRun = false; + } + } public FSImage getFSImage() { return dir.fsImage; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java index cc9552aec2..a158a5ed6b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.namenode.ha; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY; import static org.junit.Assert.*; import java.io.File; @@ -127,6 +129,7 @@ public void testSharedDirsComeFirstInEditsList() throws Exception { @Test public void testFailureOfSharedDir() throws Exception { Configuration conf = new Configuration(); + conf.setLong(DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY, 2000); // The shared edits dir will automatically be marked required. MiniDFSCluster cluster = null; @@ -151,6 +154,15 @@ public void testFailureOfSharedDir() throws Exception { assertEquals(0, FileUtil.chmod(sharedEditsDir.getAbsolutePath(), "-w", true)); + Thread.sleep(conf.getLong(DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY, + DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT) * 2); + + NameNode nn1 = cluster.getNameNode(1); + assertTrue(nn1.isStandbyState()); + assertFalse( + "StandBy NameNode should not go to SafeMode on resource unavailability", + nn1.isInSafeMode()); + NameNode nn0 = cluster.getNameNode(0); nn0.getNamesystem().getFSImage().getEditLog().getJournalSet() .setRuntimeForTesting(mockRuntime); From f35e1d49fe7e517f56da2c79de18f7b8c63516d4 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Fri, 8 Jun 2012 15:47:50 +0000 Subject: [PATCH 14/91] MAPREDUCE-3842. Stop webpages from automatic refreshing (tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1348123 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../apache/hadoop/mapreduce/v2/app/webapp/CountersPage.java | 3 --- .../org/apache/hadoop/mapreduce/v2/app/webapp/JobPage.java | 3 --- .../org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java | 4 ---- .../yarn/server/nodemanager/webapp/ApplicationPage.java | 3 --- .../hadoop/yarn/server/nodemanager/webapp/ContainerPage.java | 3 --- .../hadoop/yarn/server/nodemanager/webapp/NodePage.java | 3 --- .../hadoop/yarn/server/resourcemanager/webapp/AppPage.java | 2 -- 8 files changed, 2 insertions(+), 21 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 9641fc5ce3..f6415452f9 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -568,6 +568,8 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-3350. Per-app RM page should have the list of application-attempts like on the app JHS page (Jonathon Eagles via tgraves) + MAPREDUCE-3842. Stop webpages from automatic refreshing (tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/CountersPage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/CountersPage.java index f324b535c6..d7afcd8467 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/CountersPage.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/CountersPage.java @@ -28,9 +28,6 @@ public class CountersPage extends AppView { @Override protected void preHead(Page.HTML<_> html) { commonPreHead(html); - // Counters page is a summary. Helps to refresh automatically. - html.meta_http("refresh", "10"); - String tid = $(TASK_ID); String activeNav = "3"; if(tid == null || tid.isEmpty()) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobPage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobPage.java index fd98549991..00f4750fd3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobPage.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobPage.java @@ -33,9 +33,6 @@ public class JobPage extends AppView { : join("MapReduce Job ", $(JOB_ID))); commonPreHead(html); - // This is a job-summary page. Helps to refresh automatically. - html.meta_http("refresh", "10"); - set(initID(ACCORDION, "nav"), "{autoHeight:false, active:2}"); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java index 69e114f25d..e83a957158 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java @@ -126,10 +126,6 @@ protected Collection getTaskAttempts() { @Override protected void preHead(Page.HTML<_> html) { commonPreHead(html); - // This page is a list of all attempts which are limited in number. Okay to - // refresh automatically. - html.meta_http("refresh", "10"); - set(initID(ACCORDION, "nav"), "{autoHeight:false, active:3}"); set(DATATABLES_ID, "attempts"); set(initID(DATATABLES, "attempts"), attemptsTableInit()); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java index df891d82b5..1a92491bb4 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java @@ -45,9 +45,6 @@ public class ApplicationPage extends NMView implements YarnWebParams { @Override protected void preHead(Page.HTML<_> html) { commonPreHead(html); - // Per-app information. Helps to refresh automatically. - html.meta_http("refresh", "10"); - set(DATATABLES_ID, "containers"); set(initID(DATATABLES, "containers"), containersTableInit()); setTableStyles(html, "containers"); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java index 8e117292e6..060d72ac98 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java @@ -41,9 +41,6 @@ public class ContainerPage extends NMView implements YarnWebParams { protected void preHead(Page.HTML<_> html) { commonPreHead(html); - // Per-container information. Helps to refresh automatically. - html.meta_http("refresh", "10"); - setTitle("Container " + $(CONTAINER_ID)); set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java index db13a90ada..9eb3599f82 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java @@ -42,9 +42,6 @@ public class NodePage extends NMView { protected void commonPreHead(HTML<_> html) { super.commonPreHead(html); - // Node summary page. Helps to refresh automatically. - html.meta_http("refresh", "10"); - set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java index f2abe5d6ad..a55c62f4b5 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java @@ -24,8 +24,6 @@ public class AppPage extends RmView { @Override protected void preHead(Page.HTML<_> html) { commonPreHead(html); - // App page is per-app information. Helps to refresh automatically. - html.meta_http("refresh", "10"); } @Override protected Class content() { From 01f20f17e07369eb227b65e7df7854ffe7056619 Mon Sep 17 00:00:00 2001 From: Aaron Twining Myers Date: Fri, 8 Jun 2012 19:50:24 +0000 Subject: [PATCH 15/91] HDFS-3514. Add missing TestParallelLocalRead. Contributed by Henry Robinson. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1348207 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hadoop/hdfs/TestParallelLocalRead.java | 68 +++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelLocalRead.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index beb6389ac1..d3c1ddc21c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -90,6 +90,8 @@ Trunk (unreleased changes) HDFS-3040. TestMulitipleNNDataBlockScanner is misspelled. (Madhukara Phatak via atm) + HDFS-3514. Add missing TestParallelLocalRead. (Henry Robinson via atm) + OPTIMIZATIONS HDFS-2834. Add a ByteBuffer-based read API to DFSInputStream. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelLocalRead.java new file mode 100644 index 0000000000..05702d2aa4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelLocalRead.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import java.io.IOException; + +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestParallelLocalRead extends TestParallelReadUtil { + + @BeforeClass + static public void setupCluster() throws Exception { + HdfsConfiguration conf = new HdfsConfiguration(); + + conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true); + conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, + false); + conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY, + UserGroupInformation.getCurrentUser().getShortUserName()); + + setupCluster(1, conf); + } + + @AfterClass + static public void teardownCluster() throws Exception { + TestParallelReadUtil.teardownCluster(); + } + + /** + * Do parallel read several times with different number of files and threads. + * + * Note that while this is the only "test" in a junit sense, we're actually + * dispatching a lot more. Failures in the other methods (and other threads) + * need to be manually collected, which is inconvenient. + */ + @Test + public void testParallelReadCopying() throws IOException { + runTestWorkload(new CopyingReadWorkerHelper()); + } + + @Test + public void testParallelReadByteBuffer() throws IOException { + runTestWorkload(new DirectReadWorkerHelper()); + } + + @Test + public void testParallelReadMixed() throws IOException { + runTestWorkload(new MixedWorkloadHelper()); + } +} From 6f3e57b85ca707de66965c379146f9abf1b2a673 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Fri, 8 Jun 2012 21:48:16 +0000 Subject: [PATCH 16/91] Move several JIRAs from trunk to branch-2 section of CHANGES.txt git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1348258 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 12 ++++----- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 26 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 4ac177755a..ed39a66fe4 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -9,9 +9,6 @@ Trunk (unreleased changes) NEW FEATURES - HADOOP-8135. Add ByteBufferReadable interface to FSDataInputStream. (Henry - Robinson via atm) - HADOOP-8469. Make NetworkTopology class pluggable. (Junping Du via szetszwo) @@ -55,9 +52,6 @@ Trunk (unreleased changes) HADOOP-7994. Remove getProtocolVersion and getProtocolSignature from the client side translator and server side implementation. (jitendra) - HADOOP-8244. Improve comments on ByteBufferReadable.read. (Henry Robinson - via atm) - HADOOP-7757. Test file reference count is at least 3x actual value (Jon Eagles via bobby) @@ -177,6 +171,9 @@ Branch-2 ( Unreleased changes ) HDFS-3042. Automatic failover support for NameNode HA (todd) (see dedicated section below for breakdown of subtasks) + HADOOP-8135. Add ByteBufferReadable interface to FSDataInputStream. (Henry + Robinson via atm) + IMPROVEMENTS HADOOP-8340. SNAPSHOT build versions should compare as less than their eventual @@ -199,6 +196,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8450. Remove src/test/system. (eli) + HADOOP-8244. Improve comments on ByteBufferReadable.read. (Henry Robinson + via atm) + BUG FIXES HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d3c1ddc21c..787e646e97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -90,16 +90,8 @@ Trunk (unreleased changes) HDFS-3040. TestMulitipleNNDataBlockScanner is misspelled. (Madhukara Phatak via atm) - HDFS-3514. Add missing TestParallelLocalRead. (Henry Robinson via atm) - OPTIMIZATIONS - HDFS-2834. Add a ByteBuffer-based read API to DFSInputStream. - (Henry Robinson via todd) - - HDFS-3110. Use directRead API to reduce the number of buffer copies in - libhdfs (Henry Robinson via todd) - BUG FIXES HDFS-2299. TestOfflineEditsViewer is failing on trunk. (Uma Maheswara Rao G @@ -147,11 +139,6 @@ Trunk (unreleased changes) factor is reduced after sync follwed by closing that file. (Ashish Singhi via umamahesh) - HDFS-3235. MiniDFSClusterManager doesn't correctly support -format option. - (Henry Robinson via atm) - - HDFS-3243. TestParallelRead timing out on jenkins. (Henry Robinson via todd) - HDFS-3265. PowerPc Build error. (Kumar Ravi via mattf) HDFS-2312. FSNamesystem javadoc incorrectly says its for DNs. (harsh) @@ -232,6 +219,12 @@ Branch-2 ( Unreleased changes ) HDFS-2982. Startup performance suffers when there are many edit log segments. (Colin Patrick McCabe via todd) + HDFS-2834. Add a ByteBuffer-based read API to DFSInputStream. + (Henry Robinson via todd) + + HDFS-3110. Use directRead API to reduce the number of buffer copies in + libhdfs (Henry Robinson via todd) + BUG FIXES HDFS-3385. The last block of INodeFileUnderConstruction is not @@ -305,6 +298,13 @@ Branch-2 ( Unreleased changes ) HDFS-2914. HA: Standby should not enter safemode when resources are low. (Vinay via atm) + HDFS-3235. MiniDFSClusterManager doesn't correctly support -format option. + (Henry Robinson via atm) + + HDFS-3514. Add missing TestParallelLocalRead. (Henry Robinson via atm) + + HDFS-3243. TestParallelRead timing out on jenkins. (Henry Robinson via todd) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) From 9d9e32ef403ca5ea093fce24b6aac926ebca6c2b Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Sat, 9 Jun 2012 00:54:10 +0000 Subject: [PATCH 17/91] HDFS-3490. DatanodeWebHdfsMethods throws NullPointerException if NamenodeRpcAddressParam is not set. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1348287 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../web/resources/DatanodeWebHdfsMethods.java | 4 +++ .../web/resources/NamenodeWebHdfsMethods.java | 4 +-- .../web/resources/InetSocketAddressParam.java | 4 +++ .../hadoop/hdfs/web/resources/LongParam.java | 4 +-- .../web/TestWebHdfsFileSystemContract.java | 27 +++++++++++++++++++ 6 files changed, 42 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 787e646e97..0277f829a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -305,6 +305,9 @@ Branch-2 ( Unreleased changes ) HDFS-3243. TestParallelRead timing out on jenkins. (Henry Robinson via todd) + HDFS-3490. DatanodeWebHdfsMethods throws NullPointerException if + NamenodeRpcAddressParam is not set. (szetszwo) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java index 32b5c60cc6..eb4afe75f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java @@ -98,6 +98,10 @@ private void init(final UserGroupInformation ugi, LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path + ", ugi=" + ugi + Param.toSortedString(", ", parameters)); } + if (nnRpcAddr == null) { + throw new IllegalArgumentException(NamenodeRpcAddressParam.NAME + + " is not specified."); + } //clear content type response.setContentType(null); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 0127e959be..de8f256705 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -123,7 +123,7 @@ private void init(final UserGroupInformation ugi, final DelegationParam delegation, final UserParam username, final DoAsParam doAsUser, final UriFsPathParam path, final HttpOpParam op, - final Param... parameters) throws IOException { + final Param... parameters) { if (LOG.isTraceEnabled()) { LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path + ", ugi=" + ugi + ", " + username + ", " + doAsUser @@ -532,7 +532,7 @@ public Response getRoot( final RenewerParam renewer, @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) final BufferSizeParam bufferSize - ) throws IOException, URISyntaxException, InterruptedException { + ) throws IOException, InterruptedException { return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length, renewer, bufferSize); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/InetSocketAddressParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/InetSocketAddressParam.java index dc21f684e7..9879ba3032 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/InetSocketAddressParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/InetSocketAddressParam.java @@ -44,6 +44,10 @@ public String getDomain() { @Override InetSocketAddress parse(final String str) { + if (str == null) { + throw new IllegalArgumentException("The input string is null: expect " + + getDomain()); + } final int i = str.indexOf(':'); if (i < 0) { throw new IllegalArgumentException("Failed to parse \"" + str diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/LongParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/LongParam.java index 023402cfe0..6f102e1c9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/LongParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/LongParam.java @@ -59,7 +59,7 @@ static final class Domain extends Param.Domain { @Override public String getDomain() { - return "<" + NULL + " | short in radix " + radix + ">"; + return "<" + NULL + " | long in radix " + radix + ">"; } @Override @@ -72,7 +72,7 @@ Long parse(final String str) { } } - /** Convert a Short to a String. */ + /** Convert a Long to a String. */ String toString(final Long n) { return n == null? NULL: Long.toString(n, radix); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java index 04ffd10b35..61734180cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.web.resources.DoAsParam; import org.apache.hadoop.hdfs.web.resources.GetOpParam; import org.apache.hadoop.hdfs.web.resources.HttpOpParam; +import org.apache.hadoop.hdfs.web.resources.NamenodeRpcAddressParam; import org.apache.hadoop.hdfs.web.resources.PutOpParam; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; @@ -351,5 +352,31 @@ public void testResponseCode() throws IOException { {//test append. AppendTestUtil.testAppend(fs, new Path(dir, "append")); } + + {//test NamenodeRpcAddressParam not set. + final HttpOpParam.Op op = PutOpParam.Op.CREATE; + final URL url = webhdfs.toUrl(op, dir); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod(op.getType().toString()); + conn.setDoOutput(false); + conn.setInstanceFollowRedirects(false); + conn.connect(); + final String redirect = conn.getHeaderField("Location"); + conn.disconnect(); + + //remove NamenodeRpcAddressParam + WebHdfsFileSystem.LOG.info("redirect = " + redirect); + final int i = redirect.indexOf(NamenodeRpcAddressParam.NAME); + final int j = redirect.indexOf("&", i); + String modified = redirect.substring(0, i - 1) + redirect.substring(j); + WebHdfsFileSystem.LOG.info("modified = " + modified); + + //connect to datanode + conn = (HttpURLConnection)new URL(modified).openConnection(); + conn.setRequestMethod(op.getType().toString()); + conn.setDoOutput(op.getDoOutput()); + conn.connect(); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + } } } From 91cee8c06bbfcee78cbdf63d3dc85435aa1c5342 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Mon, 11 Jun 2012 13:54:46 +0000 Subject: [PATCH 18/91] MAPREDUCE-3927. Shuffle hang when set map.failures.percent (Bhallamudi Venkata Siva Kamesh via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1348846 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapreduce/v2/app/job/impl/JobImpl.java | 4 +- .../task/reduce/ShuffleScheduler.java | 38 ++++++----- .../task/reduce/TestShuffleScheduler.java | 67 +++++++++++++++++++ 4 files changed, 95 insertions(+), 17 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestShuffleScheduler.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index f6415452f9..b33f47d33c 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -570,6 +570,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-3842. Stop webpages from automatic refreshing (tgraves) + MAPREDUCE-3927. Shuffle hang when set map.failures.percent + (Bhallamudi Venkata Siva Kamesh via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index daafcf7c1b..220bbc31d6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -590,9 +590,9 @@ private void computeProgress() { float reduceProgress = 0f; for (Task task : this.tasks.values()) { if (task.getType() == TaskType.MAP) { - mapProgress += task.getProgress(); + mapProgress += (task.isFinished() ? 1f : task.getProgress()); } else { - reduceProgress += task.getProgress(); + reduceProgress += (task.isFinished() ? 1f : task.getProgress()); } } if (this.numMapTasks != 0) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java index df8c6b3ff0..8d5bc3b905 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java @@ -137,24 +137,26 @@ public synchronized void copySucceeded(TaskAttemptID mapId, // update the status totalBytesShuffledTillNow += bytes; - float mbs = (float) totalBytesShuffledTillNow / (1024 * 1024); - int mapsDone = totalMaps - remainingMaps; - long secsSinceStart = - (System.currentTimeMillis()-startTime)/1000+1; - - float transferRate = mbs/secsSinceStart; - progress.set((float) mapsDone / totalMaps); - String statusString = mapsDone + " / " + totalMaps + " copied."; - status.setStateString(statusString); - progress.setStatus("copy(" + mapsDone + " of " + totalMaps - + " at " + - mbpsFormat.format(transferRate) + " MB/s)"); - + updateStatus(); reduceShuffleBytes.increment(bytes); lastProgressTime = System.currentTimeMillis(); - LOG.debug("map " + mapId + " done " + statusString); + LOG.debug("map " + mapId + " done " + status.getStateString()); } } + + private void updateStatus() { + float mbs = (float) totalBytesShuffledTillNow / (1024 * 1024); + int mapsDone = totalMaps - remainingMaps; + long secsSinceStart = (System.currentTimeMillis() - startTime) / 1000 + 1; + + float transferRate = mbs / secsSinceStart; + progress.set((float) mapsDone / totalMaps); + String statusString = mapsDone + " / " + totalMaps + " copied."; + status.setStateString(statusString); + + progress.setStatus("copy(" + mapsDone + " of " + totalMaps + " at " + + mbpsFormat.format(transferRate) + " MB/s)"); + } public synchronized void copyFailed(TaskAttemptID mapId, MapHost host, boolean readError) { @@ -256,7 +258,13 @@ private void checkReducerHealth() { } public synchronized void tipFailed(TaskID taskId) { - finishedMaps[taskId.getId()] = true; + if (!finishedMaps[taskId.getId()]) { + finishedMaps[taskId.getId()] = true; + if (--remainingMaps == 0) { + notifyAll(); + } + updateStatus(); + } } public synchronized void addKnownMapOutput(String hostName, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestShuffleScheduler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestShuffleScheduler.java new file mode 100644 index 0000000000..f4ed330406 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestShuffleScheduler.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapreduce.task.reduce; + +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.TaskAttemptID; +import org.apache.hadoop.mapred.TaskStatus; +import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.mapreduce.TaskID; +import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.util.Progress; +import org.junit.Assert; +import org.junit.Test; + +public class TestShuffleScheduler { + + @SuppressWarnings("rawtypes") + @Test + public void testTipFailed() throws Exception { + JobConf job = new JobConf(); + job.setNumMapTasks(2); + + TaskStatus status = new TaskStatus() { + @Override + public boolean getIsMap() { + return false; + } + + @Override + public void addFetchFailedMap(TaskAttemptID mapTaskId) { + } + }; + Progress progress = new Progress(); + + ShuffleScheduler scheduler = new ShuffleScheduler(job, status, null, + progress, null, null, null); + + JobID jobId = new JobID(); + TaskID taskId1 = new TaskID(jobId, TaskType.REDUCE, 1); + scheduler.tipFailed(taskId1); + + Assert.assertEquals("Progress should be 0.5", 0.5f, progress.getProgress(), + 0.0f); + Assert.assertFalse(scheduler.waitUntilDone(1)); + + TaskID taskId0 = new TaskID(jobId, TaskType.REDUCE, 0); + scheduler.tipFailed(taskId0); + Assert.assertEquals("Progress should be 1.0", 1.0f, progress.getProgress(), + 0.0f); + Assert.assertTrue(scheduler.waitUntilDone(1)); + } +} From 5d80662ebba51f8d6d7fb9c563043a1e17dfe5c6 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Mon, 11 Jun 2012 18:01:38 +0000 Subject: [PATCH 19/91] HDFS-2797. Fix misuses of InputStream#skip in the edit log code. Contributed by Colin Patrick McCabe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1348945 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../namenode/EditLogFileInputStream.java | 3 +- .../hdfs/server/namenode/FSEditLogLoader.java | 5 ++++ .../hdfs/server/namenode/FSEditLogOp.java | 7 +++-- .../hdfs/server/namenode/StreamLimiter.java | 5 ++++ .../ha/TestEditLogsDuringFailover.java | 30 +++++++++++++++++-- 6 files changed, 48 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0277f829a0..aa62e91d13 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -308,6 +308,9 @@ Branch-2 ( Unreleased changes ) HDFS-3490. DatanodeWebHdfsMethods throws NullPointerException if NamenodeRpcAddressParam is not set. (szetszwo) + HDFS-2797. Fix misuses of InputStream#skip in the edit log code. + (Colin Patrick McCabe via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java index e6ddf5b920..b224c863c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java @@ -165,7 +165,8 @@ private FSEditLogOp nextOpImpl(boolean skipBrokenEdits) throws IOException { LOG.warn("skipping " + skipAmt + " bytes at the end " + "of edit log '" + getName() + "': reached txid " + txId + " out of " + lastTxId); - tracker.skip(skipAmt); + tracker.clearLimit(); + IOUtils.skipFully(tracker, skipAmt); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index e1b26bb810..343472dfa1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -751,6 +751,11 @@ public void setLimit(long limit) { limitPos = curPos + limit; } + @Override + public void clearLimit() { + limitPos = Long.MAX_VALUE; + } + @Override public void mark(int limit) { super.mark(limit); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java index 489f030e13..80f637c499 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java @@ -44,6 +44,7 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.ArrayWritable; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableFactories; @@ -2289,9 +2290,11 @@ public FSEditLogOp readOp(boolean skipBrokenEdits) throws IOException { // 0xff, we want to skip over that region, because there's nothing // interesting there. long numSkip = e.getNumAfterTerminator(); - if (in.skip(numSkip) < numSkip) { + try { + IOUtils.skipFully(in, numSkip); + } catch (Throwable t) { FSImage.LOG.error("Failed to skip " + numSkip + " bytes of " + - "garbage after an OP_INVALID. Unexpected early EOF."); + "garbage after an OP_INVALID.", t); return null; } } catch (IOException e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StreamLimiter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StreamLimiter.java index 97420828d4..4e533eb0c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StreamLimiter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StreamLimiter.java @@ -27,4 +27,9 @@ interface StreamLimiter { * Set a limit. Calling this function clears any existing limit. */ public void setLimit(long limit); + + /** + * Disable limit. + */ + public void clearLimit(); } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java index 79dcec4d43..794a3b6bf3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java @@ -20,6 +20,7 @@ import static org.junit.Assert.*; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; import java.util.Collections; @@ -35,6 +36,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; import org.apache.hadoop.hdfs.server.namenode.NNStorage; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; @@ -118,8 +120,8 @@ public void testStartup() throws Exception { } } - @Test - public void testFailoverFinalizesAndReadsInProgress() throws Exception { + private void testFailoverFinalizesAndReadsInProgress( + boolean partialTxAtEnd) throws Exception { Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .nnTopology(MiniDFSNNTopology.simpleHATopology()) @@ -130,8 +132,21 @@ public void testFailoverFinalizesAndReadsInProgress() throws Exception { URI sharedUri = cluster.getSharedEditsDir(0, 1); File sharedDir = new File(sharedUri.getPath(), "current"); FSImageTestUtil.createAbortedLogWithMkdirs(sharedDir, NUM_DIRS_IN_LOG, 1); + assertEditFiles(Collections.singletonList(sharedUri), NNStorage.getInProgressEditsFileName(1)); + if (partialTxAtEnd) { + FileOutputStream outs = null; + try { + File editLogFile = + new File(sharedDir, NNStorage.getInProgressEditsFileName(1)); + outs = new FileOutputStream(editLogFile, true); + outs.write(new byte[] { 0x18, 0x00, 0x00, 0x00 } ); + LOG.error("editLogFile = " + editLogFile); + } finally { + IOUtils.cleanup(LOG, outs); + } + } // Transition one of the NNs to active cluster.transitionToActive(0); @@ -149,7 +164,18 @@ public void testFailoverFinalizesAndReadsInProgress() throws Exception { } finally { cluster.shutdown(); } + } + + @Test + public void testFailoverFinalizesAndReadsInProgressSimple() + throws Exception { + testFailoverFinalizesAndReadsInProgress(false); + } + @Test + public void testFailoverFinalizesAndReadsInProgressWithPartialTxAtEnd() + throws Exception { + testFailoverFinalizesAndReadsInProgress(true); } /** From cb107fc55e964cf767cd456ffa48545bc5b10137 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 11 Jun 2012 18:34:40 +0000 Subject: [PATCH 20/91] HADOOP-8368. Use CMake rather than autotools to build native code (ccccabe via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1348957 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/pom.xml | 103 ++----- .../hadoop-common/src/CMakeLists.txt | 126 ++++++++ .../hadoop-common/src/config.h.cmake | 10 + .../src/main/native/.autom4te.cfg | 42 --- .../hadoop-common/src/main/native/Makefile.am | 66 ----- .../src/main/native/acinclude.m4 | 28 -- .../src/main/native/configure.ac | 130 --------- .../src/main/native/lib/Makefile.am | 47 --- .../hadoop/io/compress/lz4/Lz4Compressor.c | 5 +- .../hadoop/io/compress/lz4/Lz4Decompressor.c | 5 +- .../io/compress/snappy/SnappyCompressor.c | 36 +-- .../io/compress/snappy/SnappyDecompressor.c | 36 +-- .../org_apache_hadoop_io_compress_snappy.h | 41 +-- .../hadoop/io/compress/zlib/Makefile.am | 53 ---- .../hadoop/io/compress/zlib/ZlibCompressor.c | 32 +-- .../io/compress/zlib/ZlibDecompressor.c | 32 +-- .../zlib/org_apache_hadoop_io_compress_zlib.h | 39 +-- .../org/apache/hadoop/io/nativeio/NativeIO.c | 4 +- .../src/org/apache/hadoop/util/NativeCrc32.c | 4 +- .../src/main/native/src/org_apache_hadoop.h | 17 +- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 76 +---- .../hadoop-hdfs/src/CMakeLists.txt | 126 ++++++++ .../hadoop-hdfs/src/config.h.cmake | 6 + .../src/contrib/fuse-dfs/Makefile.am | 27 -- .../src/contrib/fuse-dfs/acinclude.m4 | 270 ------------------ .../src/contrib/fuse-dfs/configure.ac | 82 ------ .../hadoop-hdfs/src/contrib/fuse-dfs/pom.xml | 164 ----------- .../src/contrib/fuse-dfs/src/CMakeLists.txt | 73 +++++ .../src/contrib/fuse-dfs/src/Makefile.am | 22 -- .../src/contrib/fuse-dfs/src/fuse_dfs.h | 8 +- .../hadoop-hdfs/src/main/native/Makefile.am | 42 --- .../hadoop-hdfs/src/main/native/configure.ac | 125 -------- .../src/main/native/m4/apfunctions.m4 | 41 --- .../hadoop-hdfs/src/main/native/m4/apjava.m4 | 142 --------- .../src/main/native/m4/apsupport.m4 | 168 ----------- hadoop-hdfs-project/pom.xml | 1 - .../hadoop-yarn-server-nodemanager/pom.xml | 64 ++--- .../src/CMakeLists.txt | 69 +++++ .../src/config.h.cmake | 6 + .../native/container-executor/.autom4te.cfg | 42 --- .../.deps/container-executor.Po | 1 - .../native/container-executor/Makefile.am | 32 --- .../native/container-executor/configure.ac | 54 ---- .../native/container-executor/impl/main.c | 5 +- 44 files changed, 511 insertions(+), 1991 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/CMakeLists.txt create mode 100644 hadoop-common-project/hadoop-common/src/config.h.cmake delete mode 100644 hadoop-common-project/hadoop-common/src/main/native/.autom4te.cfg delete mode 100644 hadoop-common-project/hadoop-common/src/main/native/Makefile.am delete mode 100644 hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 delete mode 100644 hadoop-common-project/hadoop-common/src/main/native/configure.ac delete mode 100644 hadoop-common-project/hadoop-common/src/main/native/lib/Makefile.am delete mode 100644 hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/config.h.cmake delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/Makefile.am delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/acinclude.m4 delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/configure.ac delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/pom.xml create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/CMakeLists.txt delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/Makefile.am delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/Makefile.am delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/configure.ac delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apfunctions.m4 delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apjava.m4 delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apsupport.m4 create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake delete mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.autom4te.cfg delete mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.deps/container-executor.Po delete mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/Makefile.am delete mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/configure.ac diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index a36b74d1e0..8142080613 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -536,31 +536,10 @@ /usr/local ${snappy.prefix}/lib ${snappy.prefix}/include + - - org.apache.maven.plugins - maven-antrun-plugin - - - compile - compile - - run - - - - - - - - - - - - - org.codehaus.mojo native-maven-plugin @@ -590,73 +569,27 @@ - org.codehaus.mojo - make-maven-plugin + org.apache.maven.plugins + maven-antrun-plugin - compile + make compile - - autoreconf - configure - make-install - + run + + + + + + + + + + + + - - - ${project.build.directory}/native - - -i - -f - - - - - - OS_NAME - ${os.name} - - - OS_ARCH - ${os.arch} - - - JVM_DATA_MODEL - ${sun.arch.data.model} - - - - CPPFLAGS=-I${snappy.include} - LDFLAGS=-L${snappy.lib} - - ${project.build.directory}/native - /usr/local - - - - - OS_NAME - ${os.name} - - - OS_ARCH - ${os.arch} - - - JVM_DATA_MODEL - ${sun.arch.data.model} - - - HADOOP_NATIVE_SRCDIR - ${project.build.directory}/native - - - - - ${project.build.directory}/native/target - - @@ -700,7 +633,7 @@ maven-antrun-plugin - compile + kdc compile run diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt new file mode 100644 index 0000000000..c632531180 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -0,0 +1,126 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cmake_minimum_required(VERSION 2.6 FATAL_ERROR) + +# Default to release builds +set(CMAKE_BUILD_TYPE, Release) + +# If JVM_ARCH_DATA_MODEL is 32, compile all binaries as 32-bit. +# This variable is set by maven. +if (JVM_ARCH_DATA_MODEL EQUAL 32) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") + set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32") + if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") + set(CMAKE_SYSTEM_PROCESSOR "i686") + endif () +endif (JVM_ARCH_DATA_MODEL EQUAL 32) + +# Compile a library with both shared and static variants +function(add_dual_library LIBNAME) + add_library(${LIBNAME} SHARED ${ARGN}) + add_library(${LIBNAME}_static STATIC ${ARGN}) + set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME}) +endfunction(add_dual_library) + +# Link both a static and a dynamic target against some libraries +function(target_link_dual_libraries LIBNAME) + target_link_libraries(${LIBNAME} ${ARGN}) + target_link_libraries(${LIBNAME}_static ${ARGN}) +endfunction(target_link_dual_libraries) + +function(output_directory TGT DIR) + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") +endfunction(output_directory TGT DIR) + +function(dual_output_directory TGT DIR) + output_directory(${TGT} "${DIR}") + output_directory(${TGT}_static "${DIR}") +endfunction(dual_output_directory TGT DIR) + +if (NOT GENERATED_JAVAH) + # Must identify where the generated headers have been placed + MESSAGE(FATAL_ERROR "You must set the cmake variable GENERATED_JAVAH") +endif (NOT GENERATED_JAVAH) +find_package(JNI REQUIRED) +find_package(ZLIB REQUIRED) + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64") +set(D main/native/src/org/apache/hadoop) + +GET_FILENAME_COMPONENT(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME) + +INCLUDE(CheckFunctionExists) +INCLUDE(CheckCSourceCompiles) +CHECK_FUNCTION_EXISTS(sync_file_range HAVE_SYNC_FILE_RANGE) +CHECK_FUNCTION_EXISTS(posix_fadvise HAVE_POSIX_FADVISE) + +find_library(SNAPPY_LIBRARY NAMES snappy PATHS) +find_path(SNAPPY_INCLUDE_DIR NAMES snappy.h PATHS) +if (SNAPPY_LIBRARY) + GET_FILENAME_COMPONENT(HADOOP_SNAPPY_LIBRARY ${SNAPPY_LIBRARY} NAME) + set(SNAPPY_SOURCE_FILES + "${D}/io/compress/snappy/SnappyCompressor.c" + "${D}/io/compress/snappy/SnappyDecompressor.c") +else (${SNAPPY_LIBRARY}) + set(SNAPPY_INCLUDE_DIR "") + set(SNAPPY_SOURCE_FILES "") +endif (SNAPPY_LIBRARY) + +include_directories( + ${GENERATED_JAVAH} + main/native/src + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_BINARY_DIR} + ${JNI_INCLUDE_DIRS} + ${ZLIB_INCLUDE_DIRS} + ${SNAPPY_INCLUDE_DIR} +) +CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h) + +add_dual_library(hadoop + ${D}/io/compress/lz4/Lz4Compressor.c + ${D}/io/compress/lz4/Lz4Decompressor.c + ${D}/io/compress/lz4/lz4.c + ${SNAPPY_SOURCE_FILES} + ${D}/io/compress/zlib/ZlibCompressor.c + ${D}/io/compress/zlib/ZlibDecompressor.c + ${D}/io/nativeio/NativeIO.c + ${D}/io/nativeio/errno_enum.c + ${D}/io/nativeio/file_descriptor.c + ${D}/security/JniBasedUnixGroupsMapping.c + ${D}/security/JniBasedUnixGroupsNetgroupMapping.c + ${D}/security/getGroup.c + ${D}/util/NativeCrc32.c + ${D}/util/bulk_crc32.c +) +target_link_dual_libraries(hadoop + dl + ${JAVA_JVM_LIBRARY} +) +SET(LIBHADOOP_VERSION "1.0.0") +SET_TARGET_PROPERTIES(hadoop PROPERTIES + SOVERSION ${LIBHADOOP_VERSION}) +dual_output_directory(hadoop target/usr/local/lib) diff --git a/hadoop-common-project/hadoop-common/src/config.h.cmake b/hadoop-common-project/hadoop-common/src/config.h.cmake new file mode 100644 index 0000000000..9098b68b87 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/config.h.cmake @@ -0,0 +1,10 @@ +#ifndef CONFIG_H +#define CONFIG_H + +#cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@" +#cmakedefine HADOOP_RUNAS_HOME "@HADOOP_RUNAS_HOME@" +#cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@" +#cmakedefine HAVE_SYNC_FILE_RANGE +#cmakedefine HAVE_POSIX_FADVISE + +#endif diff --git a/hadoop-common-project/hadoop-common/src/main/native/.autom4te.cfg b/hadoop-common-project/hadoop-common/src/main/native/.autom4te.cfg deleted file mode 100644 index a69c197883..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/native/.autom4te.cfg +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# autom4te configuration for hadoop-native library -# - -begin-language: "Autoheader-preselections" -args: --no-cache -end-language: "Autoheader-preselections" - -begin-language: "Automake-preselections" -args: --no-cache -end-language: "Automake-preselections" - -begin-language: "Autoreconf-preselections" -args: --no-cache -end-language: "Autoreconf-preselections" - -begin-language: "Autoconf-without-aclocal-m4" -args: --no-cache -end-language: "Autoconf-without-aclocal-m4" - -begin-language: "Autoconf" -args: --no-cache -end-language: "Autoconf" - diff --git a/hadoop-common-project/hadoop-common/src/main/native/Makefile.am b/hadoop-common-project/hadoop-common/src/main/native/Makefile.am deleted file mode 100644 index c4ca564c2b..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/native/Makefile.am +++ /dev/null @@ -1,66 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Notes: -# 1. This makefile is designed to do the actual builds in $(HADOOP_PREFIX)/build/native/${os.name}-${os-arch}. -# 2. This makefile depends on the following environment variables to function correctly: -# * HADOOP_NATIVE_SRCDIR -# * JAVA_HOME -# * JVM_DATA_MODEL -# * OS_NAME -# * OS_ARCH -# All these are setup by build.xml. -# - -# Export $(PLATFORM) to prevent proliferation of sub-shells -export PLATFORM = $(shell echo $$OS_NAME | tr [A-Z] [a-z]) - -ACLOCAL_AMFLAGS = -I m4 -AM_CPPFLAGS = @JNI_CPPFLAGS@ -I$(HADOOP_NATIVE_SRCDIR)/src \ - -I$(HADOOP_NATIVE_SRCDIR)/javah -AM_LDFLAGS = @JNI_LDFLAGS@ -AM_CFLAGS = -g -Wall -fPIC -O2 -if SPECIFY_DATA_MODEL -AM_LDFLAGS += -m$(JVM_DATA_MODEL) -AM_CFLAGS += -m$(JVM_DATA_MODEL) -endif - -lib_LTLIBRARIES = libhadoop.la -libhadoop_la_SOURCES = src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c \ - src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c \ - src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c \ - src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c \ - src/org/apache/hadoop/io/compress/lz4/lz4.c \ - src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c \ - src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c \ - src/org/apache/hadoop/security/getGroup.c \ - src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c \ - src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c \ - src/org/apache/hadoop/io/nativeio/file_descriptor.c \ - src/org/apache/hadoop/io/nativeio/errno_enum.c \ - src/org/apache/hadoop/io/nativeio/NativeIO.c \ - src/org/apache/hadoop/util/NativeCrc32.c \ - src/org/apache/hadoop/util/bulk_crc32.c - -libhadoop_la_LDFLAGS = -version-info 1:0:0 $(AM_LDFLAGS) -libhadoop_la_LIBADD = -ldl -ljvm - -# -#vim: sw=4: ts=4: noet -# diff --git a/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 b/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 deleted file mode 100644 index 93e05b8148..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 +++ /dev/null @@ -1,28 +0,0 @@ -# AC_COMPUTE_NEEDED_DSO(LIBRARY, TEST_PROGRAM, PREPROC_SYMBOL) -# -------------------------------------------------- -# Compute the 'actual' dynamic-library used -# for LIBRARY and set it to PREPROC_SYMBOL -AC_DEFUN([AC_COMPUTE_NEEDED_DSO], -[ -AC_CACHE_CHECK([Checking for the 'actual' dynamic-library for '-l$1'], ac_cv_libname_$1, - [ - echo '$2' > conftest.c - if test -z "`${CC} ${LDFLAGS} -o conftest conftest.c -l$1 2>&1`"; then - dnl Try objdump and ldd in that order to get the dynamic library - if test ! -z "`which objdump | grep -v 'no objdump'`"; then - ac_cv_libname_$1="`objdump -p conftest | grep NEEDED | grep $1 | sed 's/\W*NEEDED\W*\(.*\)\W*$/\"\1\"/'`" - elif test ! -z "`which ldd | grep -v 'no ldd'`"; then - ac_cv_libname_$1="`ldd conftest | grep $1 | sed 's/^[[[^A-Za-z0-9]]]*\([[[A-Za-z0-9\.]]]*\)[[[^A-Za-z0-9]]]*=>.*$/\"\1\"/'`" - elif test ! -z "`which otool | grep -v 'no otool'`"; then - ac_cv_libname_$1=\"`otool -L conftest | grep $1 | sed -e 's/^[ ]*//' -e 's/ .*//' -e 's/.*\/\(.*\)$/\1/'`\"; - else - AC_MSG_ERROR(Can't find either 'objdump' or 'ldd' or 'otool' to compute the dynamic library for '-l$1') - fi - else - ac_cv_libname_$1=libnotfound.so - fi - rm -f conftest* - ] -) -AC_DEFINE_UNQUOTED($3, ${ac_cv_libname_$1}, [The 'actual' dynamic-library for '-l$1']) -])# AC_COMPUTE_NEEDED_DSO diff --git a/hadoop-common-project/hadoop-common/src/main/native/configure.ac b/hadoop-common-project/hadoop-common/src/main/native/configure.ac deleted file mode 100644 index 34408d6418..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/native/configure.ac +++ /dev/null @@ -1,130 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# configure.ac for hadoop native code. -# - -# Notes: -# 1. This configure.ac depends on the following environment variables to function correctly: -# * HADOOP_NATIVE_SRCDIR -# * JAVA_HOME -# * JVM_DATA_MODEL -# * OS_NAME -# * OS_ARCH -# All these are setup by build.xml. - -# -*- Autoconf -*- -# Process this file with autoconf to produce a configure script. -# - -AC_PREREQ(2.59) -AC_INIT(src/org_apache_hadoop.h) -AC_CONFIG_SRCDIR([src/org_apache_hadoop.h]) -AC_CONFIG_AUX_DIR([config]) -AC_CONFIG_MACRO_DIR([m4]) -AC_CONFIG_HEADER([config.h]) -AC_SYS_LARGEFILE -AC_GNU_SOURCE - -AM_INIT_AUTOMAKE(hadoop,1.0.0) - -# Checks for programs. -AC_PROG_CC -AC_PROG_LIBTOOL - -# Checks for libraries. -dnl Check for '-ldl' -AC_CHECK_LIB([dl], [dlopen]) - -dnl Check for '-ljvm' -JNI_LDFLAGS="" -if test $JAVA_HOME != "" -then - JNI_LDFLAGS="-L$JAVA_HOME/jre/lib/$OS_ARCH/server" - JVMSOPATH=`find $JAVA_HOME/jre/ -name libjvm.so | head -n 1` - JNI_LDFLAGS="$JNI_LDFLAGS -L`dirname $JVMSOPATH`" -fi -LDFLAGS="$LDFLAGS $JNI_LDFLAGS" -AC_CHECK_LIB([jvm], [JNI_GetCreatedJavaVMs]) -AC_SUBST([JNI_LDFLAGS]) - -# Checks for header files. -dnl Check for Ansi C headers -AC_HEADER_STDC - -dnl Check for other standard C headers -AC_CHECK_HEADERS([stdio.h stddef.h], [], AC_MSG_ERROR(Some system headers not found... please ensure their presence on your platform.)) - -dnl Check for JNI headers -JNI_CPPFLAGS="" -if test $JAVA_HOME != "" -then - for dir in `find $JAVA_HOME/include -follow -type d` - do - JNI_CPPFLAGS="$JNI_CPPFLAGS -I$dir" - done -fi -cppflags_bak=$CPPFLAGS -CPPFLAGS="$CPPFLAGS $JNI_CPPFLAGS" -AC_CHECK_HEADERS([jni.h], [], AC_MSG_ERROR([Native java headers not found. Is \$JAVA_HOME set correctly?])) -CPPFLAGS=$cppflags_bak -AC_SUBST([JNI_CPPFLAGS]) - -dnl Check for zlib headers -AC_CHECK_HEADERS([zlib.h zconf.h], - AC_COMPUTE_NEEDED_DSO(z, - [#include "zlib.h" - int main(int argc, char **argv){zlibVersion();return 0;}], - HADOOP_ZLIB_LIBRARY), - AC_MSG_ERROR(Zlib headers were not found... native-hadoop library needs zlib to build. Please install the requisite zlib development package.)) - -dnl Check for snappy headers -AC_CHECK_HEADERS([snappy-c.h], - AC_COMPUTE_NEEDED_DSO(snappy, - [#include "snappy-c.h" - int main(int argc, char **argv){snappy_compress(0,0,0,0);return 0;}], - HADOOP_SNAPPY_LIBRARY), - AC_MSG_WARN(Snappy headers were not found... building without snappy.)) - -dnl Check for headers needed by the native Group resolution implementation -AC_CHECK_HEADERS([fcntl.h stdlib.h string.h unistd.h], [], AC_MSG_ERROR(Some system headers not found... please ensure their presence on your platform.)) - -dnl check for posix_fadvise -AC_CHECK_HEADERS(fcntl.h, [AC_CHECK_FUNCS(posix_fadvise)]) - -dnl check for sync_file_range -AC_CHECK_HEADERS(fcntl.h, [AC_CHECK_FUNCS(sync_file_range)]) - -# Checks for typedefs, structures, and compiler characteristics. -AC_C_CONST - -# Checks for library functions. -AC_CHECK_FUNCS([memset]) - -# Check for nonstandard STRERROR_R -AC_FUNC_STRERROR_R - -AM_CONDITIONAL([SPECIFY_DATA_MODEL], [case $host_cpu in arm*) false;; *) true;; esac]) - -AC_CONFIG_FILES([Makefile]) -AC_OUTPUT - -# -#vim: sw=2: ts=2: noet -# diff --git a/hadoop-common-project/hadoop-common/src/main/native/lib/Makefile.am b/hadoop-common-project/hadoop-common/src/main/native/lib/Makefile.am deleted file mode 100644 index 9b536ff440..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/native/lib/Makefile.am +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Makefile template for building libhadoop.so -# - -# -# Notes: -# 1. This makefile is designed to do the actual builds in $(HADOOP_PREFIX)/build/native/${os.name}-${os.arch}/lib -# 2. This makefile depends on the following environment variables to function correctly: -# * HADOOP_NATIVE_SRCDIR -# * JAVA_HOME -# * OS_ARCH -# All these are setup by build.xml and/or the top-level makefile. -# - -# Add .lo files in $(SUBDIRS) to construct libhadoop.so -HADOOP_OBJS = $(foreach path,$(addprefix ../,$(SUBDIRS)),$(wildcard $(path)/*.lo)) -AM_LDFLAGS = @JNI_LDFLAGS@ -if SPECIFY_DATA_MODEL -AM_LDFLAGS += -m$(JVM_DATA_MODEL) -endif - -lib_LTLIBRARIES = libhadoop.la -libhadoop_la_SOURCES = -libhadoop_la_LDFLAGS = -version-info 1:0:0 $(AM_LDFLAGS) -libhadoop_la_LIBADD = $(HADOOP_OBJS) -ldl -ljvm - -# -#vim: sw=4: ts=4: noet -# diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c index d52a4f6b2a..641ecd73b7 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c @@ -16,10 +16,7 @@ * limitations under the License. */ -#if defined HAVE_CONFIG_H - #include -#endif - +#include "config.h" #include "org_apache_hadoop.h" #include "org_apache_hadoop_io_compress_lz4_Lz4Compressor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c index 547b027cc1..3eebc1859d 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c @@ -16,10 +16,7 @@ * limitations under the License. */ -#if defined HAVE_CONFIG_H - #include -#endif - +#include "config.h" #include "org_apache_hadoop.h" #include "org_apache_hadoop_io_compress_lz4_Lz4Decompressor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c index 13991c23f4..96a2402ae7 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c @@ -16,36 +16,12 @@ * limitations under the License. */ -#if defined HAVE_CONFIG_H - #include -#endif - -#if defined HADOOP_SNAPPY_LIBRARY - -#if defined HAVE_STDIO_H - #include -#else - #error 'stdio.h not found' -#endif - -#if defined HAVE_STDLIB_H - #include -#else - #error 'stdlib.h not found' -#endif - -#if defined HAVE_STRING_H - #include -#else - #error 'string.h not found' -#endif - -#if defined HAVE_DLFCN_H - #include -#else - #error 'dlfcn.h not found' -#endif +#include +#include +#include +#include +#include "config.h" #include "org_apache_hadoop_io_compress_snappy.h" #include "org_apache_hadoop_io_compress_snappy_SnappyCompressor.h" @@ -123,5 +99,3 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso return (jint)compressed_direct_buf_len; } - -#endif //define HADOOP_SNAPPY_LIBRARY diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c index 767c5f4b31..a5f07ca556 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c @@ -16,36 +16,12 @@ * limitations under the License. */ -#if defined HAVE_CONFIG_H - #include -#endif - -#if defined HADOOP_SNAPPY_LIBRARY - -#if defined HAVE_STDIO_H - #include -#else - #error 'stdio.h not found' -#endif - -#if defined HAVE_STDLIB_H - #include -#else - #error 'stdlib.h not found' -#endif - -#if defined HAVE_STRING_H - #include -#else - #error 'string.h not found' -#endif - -#if defined HAVE_DLFCN_H - #include -#else - #error 'dlfcn.h not found' -#endif +#include +#include +#include +#include +#include "config.h" #include "org_apache_hadoop_io_compress_snappy.h" #include "org_apache_hadoop_io_compress_snappy_SnappyDecompressor.h" @@ -127,5 +103,3 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyDecompres return (jint)uncompressed_direct_buf_len; } - -#endif //define HADOOP_SNAPPY_LIBRARY diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h index 815e030673..3e99d5d20d 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h @@ -17,42 +17,13 @@ */ -#if !defined ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H +#ifndef ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H #define ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H - -#if defined HAVE_CONFIG_H - #include -#endif - -#if defined HADOOP_SNAPPY_LIBRARY - - #if defined HAVE_STDDEF_H - #include - #else - #error 'stddef.h not found' - #endif - - #if defined HAVE_SNAPPY_C_H - #include - #else - #error 'Please install snappy-development packages for your platform.' - #endif - - #if defined HAVE_DLFCN_H - #include - #else - #error "dlfcn.h not found" - #endif - - #if defined HAVE_JNI_H - #include - #else - #error 'jni.h not found' - #endif - - #include "org_apache_hadoop.h" - -#endif //define HADOOP_SNAPPY_LIBRARY +#include "org_apache_hadoop.h" +#include +#include +#include +#include #endif //ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am deleted file mode 100644 index 821f33f052..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am +++ /dev/null @@ -1,53 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Makefile template for building native 'zlib' for hadoop. -# - -# -# Notes: -# 1. This makefile is designed to do the actual builds in $(HADOOP_PREFIX)/build/native/${os.name}-${os.arch}/$(subdir) . -# 2. This makefile depends on the following environment variables to function correctly: -# * HADOOP_NATIVE_SRCDIR -# * JAVA_HOME -# * JVM_DATA_MODEL -# * OS_ARCH -# * PLATFORM -# All these are setup by build.xml and/or the top-level makefile. -# 3. The creation of requisite jni headers/stubs are also done by build.xml and they are -# assumed to be in $(HADOOP_PREFIX)/build/native/src/org/apache/hadoop/io/compress/zlib. -# - -# The 'vpath directive' to locate the actual source files -vpath %.c $(HADOOP_NATIVE_SRCDIR)/$(subdir) - -AM_CPPFLAGS = @JNI_CPPFLAGS@ -I$(HADOOP_NATIVE_SRCDIR)/src -AM_LDFLAGS = @JNI_LDFLAGS@ -AM_CFLAGS = -g -Wall -fPIC -O2 -if SPECIFY_DATA_MODEL -AM_CFLAGS += -m$(JVM_DATA_MODEL) -endif - -noinst_LTLIBRARIES = libnativezlib.la -libnativezlib_la_SOURCES = ZlibCompressor.c ZlibDecompressor.c -libnativezlib_la_LIBADD = -ldl -ljvm - -# -#vim: sw=4: ts=4: noet -# diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c index 9ada3f03b0..689c783ef7 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c @@ -16,34 +16,12 @@ * limitations under the License. */ -#if defined HAVE_CONFIG_H - #include -#endif - -#if defined HAVE_STDIO_H - #include -#else - #error 'stdio.h not found' -#endif - -#if defined HAVE_STDLIB_H - #include -#else - #error 'stdlib.h not found' -#endif - -#if defined HAVE_STRING_H - #include -#else - #error 'string.h not found' -#endif - -#if defined HAVE_DLFCN_H - #include -#else - #error 'dlfcn.h not found' -#endif +#include +#include +#include +#include +#include "config.h" #include "org_apache_hadoop_io_compress_zlib.h" #include "org_apache_hadoop_io_compress_zlib_ZlibCompressor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c index 3047dba267..6abe36381f 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c @@ -16,34 +16,12 @@ * limitations under the License. */ -#if defined HAVE_CONFIG_H - #include -#endif - -#if defined HAVE_STDIO_H - #include -#else - #error 'stdio.h not found' -#endif - -#if defined HAVE_STDLIB_H - #include -#else - #error 'stdlib.h not found' -#endif - -#if defined HAVE_STRING_H - #include -#else - #error 'string.h not found' -#endif - -#if defined HAVE_DLFCN_H - #include -#else - #error 'dlfcn.h not found' -#endif +#include +#include +#include +#include +#include "config.h" #include "org_apache_hadoop_io_compress_zlib.h" #include "org_apache_hadoop_io_compress_zlib_ZlibDecompressor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h index 16b607b4a9..c53aa531c9 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h @@ -19,40 +19,13 @@ #if !defined ORG_APACHE_HADOOP_IO_COMPRESS_ZLIB_ZLIB_H #define ORG_APACHE_HADOOP_IO_COMPRESS_ZLIB_ZLIB_H -#if defined HAVE_CONFIG_H - #include -#endif - -#if defined HAVE_STDDEF_H - #include -#else - #error 'stddef.h not found' -#endif - -#if defined HAVE_ZLIB_H - #include -#else - #error 'Please install zlib-development packages for your platform.' -#endif - -#if defined HAVE_ZCONF_H - #include -#else - #error 'Please install zlib-development packages for your platform.' -#endif - -#if defined HAVE_DLFCN_H - #include -#else - #error "dlfcn.h not found" -#endif - -#if defined HAVE_JNI_H - #include -#else - #error 'jni.h not found' -#endif +#include +#include +#include +#include +#include +#include "config.h" #include "org_apache_hadoop.h" /* A helper macro to convert the java 'stream-handle' to a z_stream pointer. */ diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index fbcf9563ee..c08ea037d9 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -16,9 +16,6 @@ * limitations under the License. */ -// get the autoconf settings -#include "config.h" - #include #include #include @@ -32,6 +29,7 @@ #include #include +#include "config.h" #include "org_apache_hadoop.h" #include "org_apache_hadoop_io_nativeio_NativeIO.h" #include "file_descriptor.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c index 869c2ba2e8..dd51c0a257 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c @@ -16,9 +16,6 @@ * limitations under the License. */ -// get the autoconf settings -#include "config.h" - #include #include #include @@ -26,6 +23,7 @@ #include #include +#include "config.h" #include "org_apache_hadoop.h" #include "org_apache_hadoop_util_NativeCrc32.h" #include "gcc_optimizations.h" diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h b/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h index 7a777c2f4f..a50c41dbbb 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h @@ -24,21 +24,10 @@ #if !defined ORG_APACHE_HADOOP_H #define ORG_APACHE_HADOOP_H -#if defined HAVE_CONFIG_H - #include -#endif - -#if defined HAVE_DLFCN_H - #include -#else - #error "dlfcn.h not found" -#endif +#include +#include -#if defined HAVE_JNI_H - #include -#else - #error 'jni.h not found' -#endif +#include "config.h" /* A helper macro to 'throw' a java exception. */ #define THROW(env, exception_name, message) \ diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index ebff0f57f5..c775c51e3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -415,76 +415,22 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> maven-antrun-plugin - compile + make compile - - run - + run - - - - + + + + + + + - - - - org.codehaus.mojo - make-maven-plugin - - - compile - compile - - autoreconf - configure - make-install - - - - ${project.build.directory}/native - - -i - -f - - - - - - ac_cv_func_malloc_0_nonnull - yes - - - JVM_ARCH - ${sun.arch.data.model} - - - - - ${project.build.directory}/native - /usr/local - - - - - ac_cv_func_malloc_0_nonnull - yes - - - JVM_ARCH - ${sun.arch.data.model} - - - - - ${project.build.directory}/native/target - - - - - - 4.0.0 - - org.apache.hadoop - hadoop-project - 3.0.0-SNAPSHOT - ../../../../../hadoop-project - - org.apache.hadoop.contrib - hadoop-hdfs-fuse - 3.0.0-SNAPSHOT - pom - - Apache Hadoop HDFS Fuse - Apache Hadoop HDFS Fuse - - - - org.apache.hadoop - hadoop-hdfs - compile - - - org.apache.hadoop - hadoop-hdfs - test - test-jar - - - - - - - - - org.apache.maven.plugins - maven-eclipse-plugin - 2.6 - - - org.apache.maven.plugins - maven-surefire-plugin - - 1 - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - - javadoc - - site - - true - true - false - ${maven.compile.source} - ${maven.compile.encoding} - - - HttpFs API - * - - - - - - - - org.apache.maven.plugins - maven-project-info-reports-plugin - - - - false - - - dependencies - - site - - - - - org.apache.rat - apache-rat-plugin - - - - - - - - - - - fuse - - false - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - prepare-compile-native - generate-sources - - run - - - - - - - - - - - compile-fuse - compile - - run - - - - - - - - - - - - - - - - diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/CMakeLists.txt new file mode 100644 index 0000000000..fb3c580e94 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/CMakeLists.txt @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Find Linux FUSE +IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + find_package(PkgConfig REQUIRED) + pkg_check_modules(FUSE fuse) + IF(FUSE_FOUND) + FLATTEN_LIST("${FUSE_CFLAGS}" " " FUSE_CFLAGS) + FLATTEN_LIST("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FUSE_CFLAGS}") + set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} ${FUSE_LDFLAGS}") + MESSAGE(STATUS "Building Linux FUSE client.") + include_directories(${FUSE_INCLUDE_DIRS}) + ELSE(FUSE_FOUND) + MESSAGE(STATUS "Failed to find Linux FUSE libraries or include files. Will not build FUSE client.") + ENDIF(FUSE_FOUND) +ELSE (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + MESSAGE(STATUS "Non-Linux system detected. Will not build FUSE client.") +ENDIF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + +IF(FUSE_FOUND) + add_executable(fuse_dfs + fuse_dfs.c + fuse_options.c + fuse_connect.c + fuse_impls_access.c + fuse_impls_chmod.c + fuse_impls_chown.c + fuse_impls_create.c + fuse_impls_flush.c + fuse_impls_getattr.c + fuse_impls_mkdir.c + fuse_impls_mknod.c + fuse_impls_open.c + fuse_impls_read.c + fuse_impls_readdir.c + fuse_impls_release.c + fuse_impls_rename.c + fuse_impls_rmdir.c + fuse_impls_statfs.c + fuse_impls_symlink.c + fuse_impls_truncate.c + fuse_impls_unlink.c + fuse_impls_utimens.c + fuse_impls_write.c + fuse_init.c + fuse_stat_struct.c + fuse_trash.c + fuse_users.c + ) + target_link_libraries(fuse_dfs + ${FUSE_LIBRARIES} + ${JAVA_JVM_LIBRARY} + hdfs + m + ) +ENDIF(FUSE_FOUND) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/Makefile.am b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/Makefile.am deleted file mode 100644 index 706297f314..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/Makefile.am +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -bin_PROGRAMS = fuse_dfs -fuse_dfs_SOURCES = fuse_dfs.c fuse_options.c fuse_trash.c fuse_stat_struct.c fuse_users.c fuse_init.c fuse_connect.c fuse_impls_access.c fuse_impls_chmod.c fuse_impls_chown.c fuse_impls_create.c fuse_impls_flush.c fuse_impls_getattr.c fuse_impls_mkdir.c fuse_impls_mknod.c fuse_impls_open.c fuse_impls_read.c fuse_impls_release.c fuse_impls_readdir.c fuse_impls_rename.c fuse_impls_rmdir.c fuse_impls_statfs.c fuse_impls_symlink.c fuse_impls_truncate.c fuse_impls_utimens.c fuse_impls_unlink.c fuse_impls_write.c -AM_CFLAGS= -Wall -g -AM_CPPFLAGS= -DPERMS=$(PERMS) -D_FILE_OFFSET_BITS=64 -I$(JAVA_HOME)/include -I$(HADOOP_PREFIX)/../../src/main/native -I$(JAVA_HOME)/include/linux -D_FUSE_DFS_VERSION=\"$(PACKAGE_VERSION)\" -DPROTECTED_PATHS=\"$(PROTECTED_PATHS)\" -I$(FUSE_HOME)/include -AM_LDFLAGS= -L$(HADOOP_PREFIX)/../../target/native/target/usr/local/lib64 -L$(HADOOP_PREFIX)/../../target/native/target/usr/local/lib -L$(FUSE_HOME)/lib -L$(JAVA_HOME)/jre/lib/$(OS_ARCH)/server -fuse_dfs_LDADD=-lfuse -lhdfs -ljvm -lm diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_dfs.h b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_dfs.h index 56ed9cb173..4554dbdbea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_dfs.h +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_dfs.h @@ -31,13 +31,9 @@ #include #include -#ifdef HAVE_CONFIG_H -#include -#endif - -#ifdef HAVE_SETXATTR #include -#endif + +#include "config.h" // // Check if a path is in the mount option supplied protected paths. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/Makefile.am b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/Makefile.am deleted file mode 100644 index 8bbd627315..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/Makefile.am +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -@PRODUCT_MK@ - -#AM_CPPFLAGS = -I$(top_srcdir) -ACLOCAL_AMFLAGS = -I m4 - -lib_LTLIBRARIES = libhdfs.la -libhdfs_la_SOURCES = hdfs.c hdfsJniHelper.c hdfs.h - -#check_PROGRAMS = hdfs_test hdfs_read hdfs_write -check_PROGRAMS = hdfs_test hdfs_read hdfs_write - -hdfs_test_SOURCES = hdfs_test.c hdfs.h -hdfs_test_LDADD = ${libdir}/libhdfs.la - -hdfs_read_SOURCES = hdfs_read.c -hdfs_read_LDADD = ${libdir}/libhdfs.la - -hdfs_write_SOURCES = hdfs_write.c -hdfs_write_LDADD = ${libdir}/libhdfs.la - -test: hdfs_test hdfs_read hdfs_write - ${LIBHDFS_SRC_DIR}/tests/test-libhdfs.sh - - -# vim: sw=4: ts=4: noet diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/configure.ac b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/configure.ac deleted file mode 100644 index d801fc4738..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/configure.ac +++ /dev/null @@ -1,125 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Autoconf input file -# $Id$ - -AC_INIT([libhdfs], [0.1.0], omalley@apache.org) -AC_PREFIX_DEFAULT([`pwd`/../install]) -AC_CONFIG_AUX_DIR([config]) - -# Generates Makefile from Makefile.am. Modify when new subdirs are added. -# Change Makefile.am also to add subdirectly. -AM_INIT_AUTOMAKE(foreign no-dist) -AC_CONFIG_FILES(Makefile) - -LT_INIT - -AC_CONFIG_MACRO_DIR([m4]) -dnl ------------------------------------------------------------------------- -dnl Check current host (forget about cross compilation) and validate it -dnl against the cache (fail if the cache differs) -dnl ------------------------------------------------------------------------- -AP_MSG_HEADER([Current host]) -AC_CANONICAL_HOST() -AP_CANONICAL_HOST_CHECK() - -dnl ------------------------------------------------------------------------- -dnl Check C environment -dnl ------------------------------------------------------------------------- -AP_MSG_HEADER([C-Language compilation tools]) -AC_PROG_CC() -AC_CHECK_TOOL(RANLIB, ranlib, :) - -dnl ------------------------------------------------------------------------- -dnl Check if this host is supported -dnl ------------------------------------------------------------------------- -AP_MSG_HEADER([Host support]) -AP_SUPPORTED_HOST() -if test "$supported_os" = "darwin" -then - if test -z "$JAVA_HOME" -a -d /System/Library/Frameworks/JavaVM.framework/Home; then - JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Home - fi - - _prevdir=`/bin/pwd` - if test -n "$JAVA_HOME" -a -d "$JAVA_HOME/include"; then - cd "$JAVA_HOME/include" - elif test -n "$JAVA_HOME" -a -d "$JAVA_HOME/../Headers"; then - cd "$JAVA_HOME/../Headers" - else - cd /System/Library/Frameworks/JavaVM.framework/Headers - fi - CFLAGS="$CFLAGS -m${JVM_ARCH} -I`/bin/pwd -P`" - cd $_prevdir - unset _prevdir -fi - -dnl ------------------------------------------------------------------------- -dnl Check JAVA environment -dnl ------------------------------------------------------------------------- -AP_MSG_HEADER([Java compilation tools]) -AP_JAVA() -AP_SABLEVM() -AP_KAFFE() -AP_PROG_JAVAC() -AP_PROG_JAR() -AP_JVM_LIBDIR() -if test "$supported_os" != "darwin" -then - case $host_cpu in - arm*) ;; - *) - CFLAGS="$CFLAGS -m${JVM_ARCH}" - LDFLAGS="$LDFLAGS -m${JVM_ARCH}" - ;; - esac - AC_MSG_RESULT([VALUE OF JVM_ARCH IS :$JVM_ARCH]) - CFLAGS="$CFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/$supported_os" - LDFLAGS="$LDFLAGS -L$LIB_JVM_DIR -ljvm -Wl,-x" -fi - -dnl ------------------------------------------------------------------------- -dnl Add gcc specific CFLAGS. -dnl ------------------------------------------------------------------------- -if test "$GCC" = "yes" -then - CFLAGS="$CFLAGS -Wall -Wstrict-prototypes" - AC_MSG_RESULT([gcc flags added]) -fi -dnl ------------------------------------------------------------------------- -dnl Add gcc specific CFLAGS. -dnl ------------------------------------------------------------------------- -if test -z "$LDCMD" -then - LDCMD="$CC" -fi -AC_SUBST(LDCMD) - - -AC_PROG_CC -AC_PROG_LIBTOOL - -AC_TYPE_SIZE_T -AC_CHECK_FUNCS([strdup strerror strtoul]) -AC_CHECK_HEADERS([fcntl.h]) -AC_C_CONST -AC_C_VOLATILE -#AC_FUNC_MALLOC -AC_HEADER_STDBOOL -AC_SUBST(PRODUCT_MK) -AC_OUTPUT diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apfunctions.m4 b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apfunctions.m4 deleted file mode 100644 index cb5938ffca..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apfunctions.m4 +++ /dev/null @@ -1,41 +0,0 @@ -dnl -dnl Licensed to the Apache Software Foundation (ASF) under one or more -dnl contributor license agreements. See the NOTICE file distributed with -dnl this work for additional information regarding copyright ownership. -dnl The ASF licenses this file to You under the Apache License, Version 2.0 -dnl (the "License"); you may not use this file except in compliance with -dnl the License. You may obtain a copy of the License at -dnl -dnl http://www.apache.org/licenses/LICENSE-2.0 -dnl -dnl Unless required by applicable law or agreed to in writing, software -dnl distributed under the License is distributed on an "AS IS" BASIS, -dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -dnl See the License for the specific language governing permissions and -dnl limitations under the License. -dnl - -dnl ------------------------------------------------------------------------- -dnl Author Pier Fumagalli -dnl Version $Id$ -dnl ------------------------------------------------------------------------- - -AC_DEFUN([AP_MSG_HEADER],[ - printf "*** %s ***\n" "$1" 1>&2 - AC_PROVIDE([$0]) -]) - -AC_DEFUN([AP_CANONICAL_HOST_CHECK],[ - AC_MSG_CHECKING([cached host system type]) - if { test x"${ac_cv_host_system_type+set}" = x"set" && - test x"$ac_cv_host_system_type" != x"$host" ; } - then - AC_MSG_RESULT([$ac_cv_host_system_type]) - AC_MSG_ERROR([remove the \"$cache_file\" file and re-run configure]) - else - AC_MSG_RESULT(ok) - ac_cv_host_system_type="$host" - fi - AC_PROVIDE([$0]) -]) - diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apjava.m4 b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apjava.m4 deleted file mode 100644 index 993fc5bed9..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apjava.m4 +++ /dev/null @@ -1,142 +0,0 @@ -dnl -dnl Licensed to the Apache Software Foundation (ASF) under one or more -dnl contributor license agreements. See the NOTICE file distributed with -dnl this work for additional information regarding copyright ownership. -dnl The ASF licenses this file to You under the Apache License, Version 2.0 -dnl (the "License"); you may not use this file except in compliance with -dnl the License. You may obtain a copy of the License at -dnl -dnl http://www.apache.org/licenses/LICENSE-2.0 -dnl -dnl Unless required by applicable law or agreed to in writing, software -dnl distributed under the License is distributed on an "AS IS" BASIS, -dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -dnl See the License for the specific language governing permissions and -dnl limitations under the License. -dnl - -dnl ------------------------------------------------------------------------- -dnl Author Pier Fumagalli -dnl Version $Id$ -dnl ------------------------------------------------------------------------- - -AC_DEFUN([AP_PROG_JAVAC_WORKS],[ - AC_CACHE_CHECK([wether the Java compiler ($JAVAC) works],ap_cv_prog_javac_works,[ - echo "public class Test {}" > Test.java - $JAVAC $JAVACFLAGS Test.java > /dev/null 2>&1 - if test $? -eq 0 - then - rm -f Test.java Test.class - ap_cv_prog_javac_works=yes - else - rm -f Test.java Test.class - AC_MSG_RESULT(no) - AC_MSG_ERROR([installation or configuration problem: javac cannot compile]) - fi - ]) -]) - -dnl Check for JAVA compilers. -AC_DEFUN([AP_PROG_JAVAC],[ - if test "$SABLEVM" != "NONE" - then - AC_PATH_PROG(JAVACSABLE,javac-sablevm,NONE,$JAVA_HOME/bin) - else - JAVACSABLE="NONE" - fi - if test "$JAVACSABLE" = "NONE" - then - XPATH="$JAVA_HOME/bin:$JAVA_HOME/Commands:$PATH" - AC_PATH_PROG(JAVAC,javac,NONE,$XPATH) - else - AC_PATH_PROG(JAVAC,javac-sablevm,NONE,$JAVA_HOME/bin) - fi - AC_MSG_RESULT([$JAVAC]) - if test "$JAVAC" = "NONE" - then - AC_MSG_ERROR([javac not found]) - fi - AP_PROG_JAVAC_WORKS() - AC_PROVIDE([$0]) - AC_SUBST(JAVAC) - AC_SUBST(JAVACFLAGS) -]) - -dnl Check for jar archivers. -AC_DEFUN([AP_PROG_JAR],[ - if test "$SABLEVM" != "NONE" - then - AC_PATH_PROG(JARSABLE,jar-sablevm,NONE,$JAVA_HOME/bin) - else - JARSABLE="NONE" - fi - if test "$JARSABLE" = "NONE" - then - XPATH="$JAVA_HOME/bin:$JAVA_HOME/Commands:$PATH" - AC_PATH_PROG(JAR,jar,NONE,$XPATH) - else - AC_PATH_PROG(JAR,jar-sablevm,NONE,$JAVA_HOME/bin) - fi - if test "$JAR" = "NONE" - then - AC_MSG_ERROR([jar not found]) - fi - AC_PROVIDE([$0]) - AC_SUBST(JAR) -]) - -AC_DEFUN([AP_JAVA],[ - AC_ARG_WITH(java,[ --with-java=DIR Specify the location of your JDK installation],[ - AC_MSG_CHECKING([JAVA_HOME]) - if test -d "$withval" - then - JAVA_HOME="$withval" - AC_MSG_RESULT([$JAVA_HOME]) - else - AC_MSG_RESULT([failed]) - AC_MSG_ERROR([$withval is not a directory]) - fi - AC_SUBST(JAVA_HOME) - ]) - if test x"$JAVA_HOME" = x - then - AC_MSG_ERROR([Java Home not defined. Rerun with --with-java=[...] parameter]) - fi -]) - -dnl check if the JVM in JAVA_HOME is sableVM -dnl $JAVA_HOME/bin/sablevm and /opt/java/lib/sablevm/bin are tested. -AC_DEFUN([AP_SABLEVM],[ - if test x"$JAVA_HOME" != x - then - AC_PATH_PROG(SABLEVM,sablevm,NONE,$JAVA_HOME/bin) - if test "$SABLEVM" = "NONE" - then - dnl java may be SableVM. - if $JAVA_HOME/bin/java -version 2> /dev/null | grep SableVM > /dev/null - then - SABLEVM=$JAVA_HOME/bin/java - fi - fi - if test "$SABLEVM" != "NONE" - then - AC_MSG_RESULT([Using sableVM: $SABLEVM]) - CFLAGS="$CFLAGS -DHAVE_SABLEVM" - fi - fi -]) - -dnl check if the JVM in JAVA_HOME is kaffe -dnl $JAVA_HOME/bin/kaffe is tested. -AC_DEFUN([AP_KAFFE],[ - if test x"$JAVA_HOME" != x - then - AC_PATH_PROG(KAFFEVM,kaffe,NONE,$JAVA_HOME/bin) - if test "$KAFFEVM" != "NONE" - then - AC_MSG_RESULT([Using kaffe: $KAFFEVM]) - CFLAGS="$CFLAGS -DHAVE_KAFFEVM" - LDFLAGS="$LDFLAGS -Wl,-rpath $JAVA_HOME/jre/lib/$HOST_CPU -L $JAVA_HOME/jre/lib/$HOST_CPU -lkaffevm" - fi - fi -]) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apsupport.m4 b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apsupport.m4 deleted file mode 100644 index 0c8b262dcb..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/m4/apsupport.m4 +++ /dev/null @@ -1,168 +0,0 @@ -dnl -dnl Licensed to the Apache Software Foundation (ASF) under one or more -dnl contributor license agreements. See the NOTICE file distributed with -dnl this work for additional information regarding copyright ownership. -dnl The ASF licenses this file to You under the Apache License, Version 2.0 -dnl (the "License"); you may not use this file except in compliance with -dnl the License. You may obtain a copy of the License at -dnl -dnl http://www.apache.org/licenses/LICENSE-2.0 -dnl -dnl Unless required by applicable law or agreed to in writing, software -dnl distributed under the License is distributed on an "AS IS" BASIS, -dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -dnl See the License for the specific language governing permissions and -dnl limitations under the License. -dnl - -dnl ------------------------------------------------------------------------- -dnl Author Pier Fumagalli -dnl Version $Id$ -dnl ------------------------------------------------------------------------- - -AC_DEFUN([AP_SUPPORTED_HOST],[ - AC_MSG_CHECKING([C flags dependant on host system type]) - - case $host_os in - darwin*) - CFLAGS="$CFLAGS -DOS_DARWIN -DDSO_DYLD" - supported_os="darwin" - ;; - solaris*) - CFLAGS="$CFLAGS -DOS_SOLARIS -DDSO_DLFCN" - supported_os="solaris" - LIBS="$LIBS -ldl -lthread" - ;; - linux*) - CFLAGS="$CFLAGS -DOS_LINUX -DDSO_DLFCN" - supported_os="linux" - LIBS="$LIBS -ldl -lpthread" - ;; - cygwin) - CFLAGS="$CFLAGS -DOS_CYGWIN -DDSO_DLFCN -DNO_SETSID" - supported_os="win32" - ;; - sysv) - CFLAGS="$CFLAGS -DOS_SYSV -DDSO_DLFCN" - LIBS="$LIBS -ldl" - ;; - sysv4) - CFLAGS="$CFLAGS -DOS_SYSV -DDSO_DLFCN -Kthread" - LDFLAGS="-Kthread $LDFLAGS" - LIBS="$LIBS -ldl" - ;; - freebsd*) - CFLAGS="$CFLAGS -DOS_FREEBSD -DDSO_DLFCN -D_THREAD_SAFE -pthread" - LDFLAGS="-pthread $LDFLAGS" - supported_os="freebsd" - ;; - osf5*) - CFLAGS="$CFLAGS -pthread -DOS_TRU64 -DDSO_DLFCN -D_XOPEN_SOURCE_EXTENDED" - LDFLAGS="$LDFLAGS -pthread" - ;; - hpux11*) - CFLAGS="$CFLAGS -pthread -DOS_HPUX -DDSO_DLFCN" - LDFLAGS="$LDFLAGS -pthread" - LIBS="$LIBS -lpthread" - ;; - *) - AC_MSG_RESULT([failed]) - AC_MSG_ERROR([Unsupported operating system "$host_os"]);; - esac - - case $host_cpu in - powerpc*) - CFLAGS="$CFLAGS -DCPU=\\\"$host_cpu\\\"" - HOST_CPU=$host_cpu;; - sparc*) - CFLAGS="$CFLAGS -DCPU=\\\"$host_cpu\\\"" - HOST_CPU=$host_cpu;; - i?86) - CFLAGS="$CFLAGS -DCPU=\\\"i386\\\"" - HOST_CPU=i386;; - x86_64) - CFLAGS="$CFLAGS -DCPU=\\\"amd64\\\"" - HOST_CPU=amd64;; - bs2000) - CFLAGS="$CFLAGS -DCPU=\\\"osd\\\" -DCHARSET_EBCDIC -DOSD_POSIX" - supported_os="osd" - LDFLAGS="-Kno_link_stdlibs -B llm4" - LIBS="$LIBS -lBLSLIB" - LDCMD="/opt/C/bin/cc" - HOST_CPU=osd;; - mips) - CFLAGS="$CFLAGS -DCPU=\\\"mips\\\"" - supported_os="mips" - HOST_CPU=mips;; - alpha*) - CFLAGS="$CFLAGS -DCPU=\\\"alpha\\\"" - supported_os="alpha" - HOST_CPU=alpha;; - hppa2.0w) - CFLAGS="$CFLAGS -DCPU=\\\"PA_RISC2.0W\\\"" - supported_os="hp-ux" - HOST_CPU=PA_RISC2.0W;; - hppa2.0) - CFLAGS="$CFLAGS -DCPU=\\\"PA_RISC2.0\\\"" - supported_os="hp-ux" - HOST_CPU=PA_RISC2.0;; - mipsel) - CFLAGS="$CFLAGS -DCPU=\\\"mipsel\\\"" - supported_os="mipsel" - HOST_CPU=mipsel;; - ia64) - CFLAGS="$CFLAGS -DCPU=\\\"ia64\\\"" - supported_os="ia64" - HOST_CPU=ia64;; - s390) - CFLAGS="$CFLAGS -DCPU=\\\"s390\\\"" - supported_os="s390" - HOST_CPU=s390;; - arm*) - CFLAGS="$CFLAGS -DCPU=\\\"arm\\\"" - supported_os="arm" - HOST_CPU=arm;; - *) - AC_MSG_RESULT([failed]) - AC_MSG_ERROR([Unsupported CPU architecture "$host_cpu"]);; - esac - - AC_MSG_RESULT([ok]) - AC_SUBST(CFLAGS) - AC_SUBST(LDFLAGS) -]) - -AC_DEFUN([AP_JVM_LIBDIR],[ - AC_MSG_CHECKING([where on earth this jvm library is..]) - javabasedir=$JAVA_HOME - case $host_os in - cygwin* | mingw* | pw23* ) - lib_jvm_dir=`find $javabasedir -follow \( \ - \( -name client -type d -prune \) -o \ - \( -name "jvm.dll" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` - ;; - aix*) - lib_jvm_dir=`find $javabasedir \( \ - \( -name client -type d -prune \) -o \ - \( -name "libjvm.*" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` - if test -z "$lib_jvm_dir"; then - lib_jvm_dir=`find $javabasedir \( \ - \( -name client -type d -prune \) -o \ - \( -name "libkaffevm.*" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` - fi - ;; - *) - lib_jvm_dir=`find $javabasedir -follow \( \ - \( -name client -type d -prune \) -o \ - \( -name "libjvm.*" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` - if test -z "$lib_jvm_dir"; then - lib_jvm_dir=`find $javabasedir -follow \( \ - \( -name client -type d -prune \) -o \ - \( -name "libkaffevm.*" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` - fi - ;; - esac - LIB_JVM_DIR=$lib_jvm_dir - AC_MSG_RESULT([ohh u there ... $LIB_JVM_DIR]) - AC_SUBST(LIB_JVM_DIR) -]) diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml index 0e2684b1ea..27161004a3 100644 --- a/hadoop-hdfs-project/pom.xml +++ b/hadoop-hdfs-project/pom.xml @@ -34,7 +34,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> hadoop-hdfs hadoop-hdfs-httpfs hadoop-hdfs/src/contrib/bkjournal - hadoop-hdfs/src/contrib/fuse-dfs diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index fb3c97b521..f865b2dc59 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -47,47 +47,37 @@ - org.codehaus.mojo - make-maven-plugin + org.apache.maven.plugins + maven-antrun-plugin - compile + make compile - - autoreconf - configure - make-install - + run + + + + + + + + + + + + - test + native_tests test - - test - + + + + + + - - - ${project.build.directory}/native/container-executor - - -i - - - - - - CFLAGS - -DHADOOP_CONF_DIR=${container-executor.conf.dir} ${container-executor.additional_cflags} - - - ${project.build.directory}/native/container-executor - /usr/local - - - ${project.build.directory}/native/target - - @@ -172,14 +162,6 @@ run - - - - - - - - diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt new file mode 100644 index 0000000000..ace151a68b --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 2.6 FATAL_ERROR) + +set(CMAKE_BUILD_TYPE, Release) + +if (JVM_ARCH_DATA_MODEL EQUAL 32) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") + set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32") + if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") + set(CMAKE_SYSTEM_PROCESSOR "i686") + endif () +endif (JVM_ARCH_DATA_MODEL EQUAL 32) + +function(output_directory TGT DIR) + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") +endfunction(output_directory TGT DIR) + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2 -D_GNU_SOURCE") +# note: can't enable -D_LARGEFILE: see MAPREDUCE-4258 +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT") + +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_BINARY_DIR} + main/native/container-executor + main/native/container-executor/impl +) +CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h) + +add_library(container + main/native/container-executor/impl/configuration.c + main/native/container-executor/impl/container-executor.c +) + +add_executable(container-executor + main/native/container-executor/impl/main.c +) +target_link_libraries(container-executor + container +) +output_directory(container-executor target/usr/local/bin) + +add_executable(test-container-executor + main/native/container-executor/test/test-container-executor.c +) +target_link_libraries(test-container-executor + container +) +output_directory(test-container-executor target/usr/local/bin) diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake new file mode 100644 index 0000000000..1fff36131f --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake @@ -0,0 +1,6 @@ +#ifndef CONFIG_H +#define CONFIG_H + +#cmakedefine HADOOP_CONF_DIR "@HADOOP_CONF_DIR@" + +#endif diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.autom4te.cfg b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.autom4te.cfg deleted file mode 100644 index d21d1c9877..0000000000 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.autom4te.cfg +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# autom4te configuration for hadoop utils library -# - -begin-language: "Autoheader-preselections" -args: --no-cache -end-language: "Autoheader-preselections" - -begin-language: "Automake-preselections" -args: --no-cache -end-language: "Automake-preselections" - -begin-language: "Autoreconf-preselections" -args: --no-cache -end-language: "Autoreconf-preselections" - -begin-language: "Autoconf-without-aclocal-m4" -args: --no-cache -end-language: "Autoconf-without-aclocal-m4" - -begin-language: "Autoconf" -args: --no-cache -end-language: "Autoconf" - diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.deps/container-executor.Po b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.deps/container-executor.Po deleted file mode 100644 index 9ce06a81ea..0000000000 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/.deps/container-executor.Po +++ /dev/null @@ -1 +0,0 @@ -# dummy diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/Makefile.am b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/Makefile.am deleted file mode 100644 index 4938bb2f53..0000000000 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/Makefile.am +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -AM_CFLAGS=-I$(srcdir)/impl -Wall -g -Werror - -# Define the programs that need to be built -bin_PROGRAMS = container-executor -check_PROGRAMS = test-container-executor - -TESTS = test-container-executor - -# Define the sources for the common files -common_SOURCES = impl/configuration.c impl/container-executor.c - -# Define the sources for the real executable -container_executor_SOURCES = $(common_SOURCES) impl/main.c - -# Define the sources for the test executable -test_container_executor_SOURCES = $(common_SOURCES) test/test-container-executor.c diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/configure.ac b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/configure.ac deleted file mode 100644 index db8af88cf1..0000000000 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/configure.ac +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -*- Autoconf -*- -# Process this file with autoconf to produce a configure script. - -AC_PREREQ(2.59) -AC_INIT(linux-container-executor, 1.0.0, mapreduce-dev@hadoop.apache.org) -AC_GNU_SOURCE -#AC_SYS_LARGEFILE - -AM_INIT_AUTOMAKE([subdir-objects foreign no-dist]) - -AC_CONFIG_SRCDIR([impl/container-executor.c]) -AC_CONFIG_FILES([Makefile]) - -AC_PREFIX_DEFAULT(`pwd`/../install) - -CHECK_INSTALL_CFLAG -HADOOP_UTILS_SETUP - -# Checks for programs. -AC_PROG_CC -AM_PROG_CC_C_O - -# Checks for libraries. - -# Checks for header files. -AC_LANG(C) -AC_CHECK_HEADERS([unistd.h]) - -# Checks for typedefs, structures, and compiler characteristics. -AC_HEADER_STDBOOL -AC_C_CONST -AC_TYPE_OFF_T -AC_TYPE_SIZE_T -AC_FUNC_STRERROR_R - -# Checks for library functions. -AC_CHECK_FUNCS([mkdir uname]) -AC_OUTPUT diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index cd8caabe33..d6ce5aa706 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -16,6 +16,7 @@ * limitations under the License. */ +#include "config.h" #include "configuration.h" #include "container-executor.h" @@ -29,8 +30,6 @@ #include #include -#define _STRINGIFY(X) #X -#define STRINGIFY(X) _STRINGIFY(X) #define CONF_FILENAME "container-executor.cfg" // When building as part of a Maven build this value gets defined by using @@ -101,7 +100,7 @@ int main(int argc, char **argv) { char *executable_file = get_executable(); - char *orig_conf_file = STRINGIFY(HADOOP_CONF_DIR) "/" CONF_FILENAME; + char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME; char *conf_file = resolve_config_path(orig_conf_file, argv[0]); char *local_dirs, *log_dirs; From 40223a3288d88d67c7f028aa8abb044ddb97c6e2 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 11 Jun 2012 18:47:18 +0000 Subject: [PATCH 21/91] HADOOP-8368. Amendment to add entry in CHANGES.txt git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1348960 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ed39a66fe4..7b9332d954 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -199,6 +199,8 @@ Branch-2 ( Unreleased changes ) HADOOP-8244. Improve comments on ByteBufferReadable.read. (Henry Robinson via atm) + HADOOP-8368. Use CMake rather than autotools to build native code (ccccabe via tucu) + BUG FIXES HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname From cede63d6929a8387eeaa13ef889d4ac180144366 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Mon, 11 Jun 2012 20:37:03 +0000 Subject: [PATCH 22/91] MAPREDUCE-3871. Allow symlinking in LocalJobRunner DistributedCache. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1348997 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapred/LocalDistributedCacheManager.java | 59 ++++++++++++++++--- .../mapred/TestMRWithDistributedCache.java | 36 ++++++----- 3 files changed, 74 insertions(+), 24 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index b33f47d33c..7c7897425b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -129,6 +129,9 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-4146. Support limits on task status string length and number of block locations in branch-2. (Ahmed Radwan via tomwhite) + MAPREDUCE-3871. Allow symlinking in LocalJobRunner DistributedCache. + (tomwhite) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java index 2fec8147e3..670959b8c4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLClassLoader; @@ -45,6 +46,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.MRConfig; @@ -72,6 +74,8 @@ class LocalDistributedCacheManager { private List localFiles = new ArrayList(); private List localClasspaths = new ArrayList(); + private List symlinksCreated = new ArrayList(); + private boolean setupCalled = false; /** @@ -172,18 +176,51 @@ public void setup(JobConf conf) throws IOException { .size()]))); } if (DistributedCache.getSymlink(conf)) { - // This is not supported largely because, - // for a Child subprocess, the cwd in LocalJobRunner - // is not a fresh slate, but rather the user's working directory. - // This is further complicated because the logic in - // setupWorkDir only creates symlinks if there's a jarfile - // in the configuration. - LOG.warn("LocalJobRunner does not support " + - "symlinking into current working dir."); + File workDir = new File(System.getProperty("user.dir")); + URI[] archives = DistributedCache.getCacheArchives(conf); + URI[] files = DistributedCache.getCacheFiles(conf); + Path[] localArchives = DistributedCache.getLocalCacheArchives(conf); + Path[] localFiles = DistributedCache.getLocalCacheFiles(conf); + if (archives != null) { + for (int i = 0; i < archives.length; i++) { + String link = archives[i].getFragment(); + String target = new File(localArchives[i].toUri()).getPath(); + symlink(workDir, target, link); + } + } + if (files != null) { + for (int i = 0; i < files.length; i++) { + String link = files[i].getFragment(); + String target = new File(localFiles[i].toUri()).getPath(); + symlink(workDir, target, link); + } + } } setupCalled = true; } + /** + * Utility method for creating a symlink and warning on errors. + * + * If link is null, does nothing. + */ + private void symlink(File workDir, String target, String link) + throws IOException { + if (link != null) { + link = workDir.toString() + Path.SEPARATOR + link; + File flink = new File(link); + if (!flink.exists()) { + LOG.info(String.format("Creating symlink: %s <- %s", target, link)); + if (0 != FileUtil.symLink(target, link)) { + LOG.warn(String.format("Failed to create symlink: %s <- %s", target, + link)); + } else { + symlinksCreated.add(new File(link)); + } + } + } + } + /** * Are the resources that should be added to the classpath? * Should be called after setup(). @@ -217,6 +254,12 @@ public ClassLoader run() { } public void close() throws IOException { + for (File symlink : symlinksCreated) { + if (!symlink.delete()) { + LOG.warn("Failed to delete symlink created by the local job runner: " + + symlink); + } + } FileContext localFSFileContext = FileContext.getLocalFSFileContext(); for (String archive : localArchives) { localFSFileContext.delete(new Path(archive), true); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java index 6798831ba7..4336807821 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.util.Arrays; import java.util.jar.JarOutputStream; import java.util.zip.ZipEntry; @@ -61,6 +62,9 @@ public class TestMRWithDistributedCache extends TestCase { private static Path TEST_ROOT_DIR = new Path(System.getProperty("test.build.data","/tmp")); + private static File symlinkFile = new File("distributed.first.symlink"); + private static File expectedAbsentSymlinkFile = + new File("distributed.second.jar"); private static Configuration conf = new Configuration(); private static FileSystem localFs; static { @@ -107,20 +111,17 @@ public void setup(Context context) throws IOException { TestCase.assertNotNull(cl.getResource("distributed.jar.inside3")); TestCase.assertNull(cl.getResource("distributed.jar.inside4")); - // Check that the symlink for the renaming was created in the cwd; - // This only happens for real for non-local jobtrackers. - // (The symlinks exist in "localRunner/" for local Jobtrackers, - // but the user has no way to get at them. - if (!"local".equals( - context.getConfiguration().get(JTConfig.JT_IPC_ADDRESS))) { - File symlinkFile = new File("distributed.first.symlink"); - TestCase.assertTrue("symlink distributed.first.symlink doesn't exist", symlinkFile.exists()); - TestCase.assertEquals("symlink distributed.first.symlink length not 1", 1, symlinkFile.length()); - } + TestCase.assertTrue("symlink distributed.first.symlink doesn't exist", + symlinkFile.exists()); + TestCase.assertEquals("symlink distributed.first.symlink length not 1", 1, + symlinkFile.length()); + + TestCase.assertFalse("second file should not be symlinked", + expectedAbsentSymlinkFile.exists()); } } - + private void testWithConf(Configuration conf) throws IOException, InterruptedException, ClassNotFoundException, URISyntaxException { // Create a temporary file of length 1. @@ -144,11 +145,7 @@ private void testWithConf(Configuration conf) throws IOException, job.addFileToClassPath(second); job.addArchiveToClassPath(third); job.addCacheArchive(fourth.toUri()); - - // don't create symlink for LocalJobRunner - if (!"local".equals(conf.get(JTConfig.JT_IPC_ADDRESS))) { - job.createSymlink(); - } + job.createSymlink(); job.setMaxMapAttempts(1); // speed up failures job.submit(); @@ -157,10 +154,17 @@ private void testWithConf(Configuration conf) throws IOException, /** Tests using the local job runner. */ public void testLocalJobRunner() throws Exception { + symlinkFile.delete(); // ensure symlink is not present (e.g. if test is + // killed part way through) + Configuration c = new Configuration(); c.set(JTConfig.JT_IPC_ADDRESS, "local"); c.set("fs.defaultFS", "file:///"); testWithConf(c); + + assertFalse("Symlink not removed by local job runner", + // Symlink target will have gone so can't use File.exists() + Arrays.asList(new File(".").list()).contains(symlinkFile.getName())); } private Path createTempFile(String filename, String contents) From d2c6c0c09909d3e57a444820e200e1f416b95c9c Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Mon, 11 Jun 2012 20:38:46 +0000 Subject: [PATCH 23/91] HDFS-3052. Change INodeFile and INodeFileUnderConstruction to package private. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1348998 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../server/blockmanagement/BlockInfo.java | 5 ++-- .../hdfs/server/namenode/FSDirectory.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 22 ++++++++-------- .../hdfs/server/namenode/INodeFile.java | 11 ++------ .../namenode/INodeFileUnderConstruction.java | 7 +++--- .../apache/hadoop/hdfs/TestPersistBlocks.java | 25 ++++++++----------- .../blockmanagement/TestBlockManager.java | 15 +++++------ 8 files changed, 39 insertions(+), 51 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index aa62e91d13..f916805d09 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -214,6 +214,9 @@ Branch-2 ( Unreleased changes ) HDFS-1013. Miscellaneous improvements to HTML markup for web UIs (Eugene Koontz via todd) + HDFS-3052. Change INodeFile and INodeFileUnderConstruction to package + private. (szetszwo) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java index be86b536c3..0739aab970 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -30,8 +30,9 @@ * the block are stored. */ @InterfaceAudience.Private -public class BlockInfo extends Block implements - LightWeightGSet.LinkedElement { +public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { + public static final BlockInfo[] EMPTY_ARRAY = {}; + private BlockCollection bc; /** For implementing {@link LightWeightGSet.LinkedElement} interface */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index d4239288a4..f5664a114a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -277,7 +277,7 @@ INode unprotectedAddFile( String path, preferredBlockSize, modificationTime, clientName, clientMachine, null); } else { - newNode = new INodeFile(permissions, 0, replication, + newNode = new INodeFile(permissions, BlockInfo.EMPTY_ARRAY, replication, modificationTime, atime, preferredBlockSize); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 386b5eb467..22013e0feb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1605,7 +1605,7 @@ private LocatedBlock startFileInternal(String src, } try { - INode myFile = dir.getFileINode(src); + INodeFile myFile = dir.getFileINode(src); recoverLeaseInternal(myFile, src, holder, clientMachine, false); try { @@ -1681,22 +1681,20 @@ private LocatedBlock startFileInternal(String src, * @throws UnresolvedLinkException * @throws IOException */ - public LocatedBlock prepareFileForWrite(String src, INode file, + LocatedBlock prepareFileForWrite(String src, INodeFile file, String leaseHolder, String clientMachine, DatanodeDescriptor clientNode, - boolean writeToEditLog) - throws UnresolvedLinkException, IOException { - INodeFile node = (INodeFile) file; + boolean writeToEditLog) throws IOException { INodeFileUnderConstruction cons = new INodeFileUnderConstruction( - node.getLocalNameBytes(), - node.getReplication(), - node.getModificationTime(), - node.getPreferredBlockSize(), - node.getBlocks(), - node.getPermissionStatus(), + file.getLocalNameBytes(), + file.getReplication(), + file.getModificationTime(), + file.getPreferredBlockSize(), + file.getBlocks(), + file.getPermissionStatus(), leaseHolder, clientMachine, clientNode); - dir.replaceNode(src, node, cons); + dir.replaceNode(src, file, cons); leaseManager.addLease(cons.getClientName(), src); LocatedBlock ret = blockManager.convertLastBlockToUnderConstruction(cons); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 3bfb335906..a6df657c8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -25,13 +25,13 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection; /** I-node for closed file. */ @InterfaceAudience.Private -public class INodeFile extends INode implements BlockCollection { +class INodeFile extends INode implements BlockCollection { static final FsPermission UMASK = FsPermission.createImmutable((short)0111); //Number of bits for Block size @@ -45,13 +45,6 @@ public class INodeFile extends INode implements BlockCollection { BlockInfo blocks[] = null; - INodeFile(PermissionStatus permissions, - int nrBlocks, short replication, long modificationTime, - long atime, long preferredBlockSize) { - this(permissions, new BlockInfo[nrBlocks], replication, - modificationTime, atime, preferredBlockSize); - } - INodeFile(PermissionStatus permissions, BlockInfo[] blklist, short replication, long modificationTime, long atime, long preferredBlockSize) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java index 66e33e077d..1c19a1be83 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java @@ -19,6 +19,7 @@ import java.io.IOException; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; @@ -32,8 +33,8 @@ /** * I-node for file being written. */ -public class INodeFileUnderConstruction extends INodeFile - implements MutableBlockCollection { +@InterfaceAudience.Private +class INodeFileUnderConstruction extends INodeFile implements MutableBlockCollection { private String clientName; // lease holder private final String clientMachine; private final DatanodeDescriptor clientNode; // if client is a cluster node too. @@ -45,7 +46,7 @@ public class INodeFileUnderConstruction extends INodeFile String clientName, String clientMachine, DatanodeDescriptor clientNode) { - super(permissions.applyUMask(UMASK), 0, replication, + super(permissions.applyUMask(UMASK), BlockInfo.EMPTY_ARRAY, replication, modTime, modTime, preferredBlockSize); this.clientName = clientName; this.clientMachine = clientMachine; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java index cb989298fa..497d29d738 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java @@ -18,39 +18,34 @@ package org.apache.hadoop.hdfs; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.Random; + import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; -import org.apache.hadoop.hdfs.server.namenode.FSEditLog; import org.apache.hadoop.hdfs.server.namenode.FSImage; -import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; -import org.apache.hadoop.hdfs.server.namenode.INodeFileUnderConstruction; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.util.Collection; -import java.util.List; -import java.util.Random; -import static org.junit.Assert.*; import org.junit.Test; -import com.google.common.collect.Lists; - /** * A JUnit test for checking if restarting DFS preserves the * blocks that are part of an unclosed file. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index 743fb3b08d..07682fe9b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -17,7 +17,9 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -29,14 +31,9 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; -import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.net.NetworkTopology; import org.junit.Before; import org.junit.Test; @@ -381,11 +378,11 @@ private List startDecommission(int ... indexes) { } private BlockInfo addBlockOnNodes(long blockId, List nodes) { - INodeFile iNode = Mockito.mock(INodeFile.class); - Mockito.doReturn((short)3).when(iNode).getReplication(); + BlockCollection bc = Mockito.mock(BlockCollection.class); + Mockito.doReturn((short)3).when(bc).getReplication(); BlockInfo blockInfo = blockOnNodes(blockId, nodes); - bm.blocksMap.addBlockCollection(blockInfo, iNode); + bm.blocksMap.addBlockCollection(blockInfo, bc); return blockInfo; } From fa4052fd302e5328735f37988e9fe25476ac951b Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Mon, 11 Jun 2012 20:52:45 +0000 Subject: [PATCH 24/91] HDFS-3517. TestStartup should bind ephemeral ports. Contributed by Eli Collins git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349004 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../org/apache/hadoop/hdfs/server/namenode/TestStartup.java | 3 +++ 2 files changed, 5 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f916805d09..26e35325d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -314,6 +314,8 @@ Branch-2 ( Unreleased changes ) HDFS-2797. Fix misuses of InputStream#skip in the edit log code. (Colin Patrick McCabe via eli) + HDFS-3517. TestStartup should bind ephemeral ports. (eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java index 74c3cf8f79..c265579427 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java @@ -100,6 +100,9 @@ protected void setUp() throws Exception { fileAsURI(new File(hdfsDir, "name")).toString()); config.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, new File(hdfsDir, "data").getPath()); + config.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, "0.0.0.0:0"); + config.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); + config.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, "0.0.0.0:0"); config.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_DIR_KEY, fileAsURI(new File(hdfsDir, "secondary")).toString()); config.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, From a92b49c6143b7d5b23e21ce117a9413435662754 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Mon, 11 Jun 2012 21:20:28 +0000 Subject: [PATCH 25/91] HADOOP-8491. Check for short writes when using FileChannel#write and related methods. Contributed by Colin Patrick McCabe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349019 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 9 ++-- .../java/org/apache/hadoop/io/IOUtils.java | 34 ++++++++++++++ .../org/apache/hadoop/io/TestIOUtils.java | 44 ++++++++++++++++++- .../namenode/EditLogFileOutputStream.java | 4 +- 4 files changed, 85 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 7b9332d954..58f39e5aae 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -242,6 +242,12 @@ Branch-2 ( Unreleased changes ) HADOOP-8485. Don't hardcode "Apache Hadoop 0.23" in the docs. (eli) + HADOOP-8488. test-patch.sh gives +1 even if the native build fails. + (Colin Patrick McCabe via eli) + + HADOOP-8491. Check for short writes when using FileChannel#write + and related methods. (Colin Patrick McCabe via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active @@ -274,9 +280,6 @@ Branch-2 ( Unreleased changes ) HADOOP-8405. ZKFC tests leak ZK instances. (todd) - HADOOP-8488. test-patch.sh gives +1 even if the native build fails. - (Colin Patrick McCabe via eli) - Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java index f5875d8d96..65d8855f14 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java @@ -20,6 +20,9 @@ import java.io.*; import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; import org.apache.commons.logging.Log; @@ -245,4 +248,35 @@ public void write(byte[] b, int off, int len) throws IOException { public void write(int b) throws IOException { } } + + /** + * Write a ByteBuffer to a WritableByteChannel, handling short writes. + * + * @param bc The WritableByteChannel to write to. + * @param buf The input buffer + * @param offset The offset in the file to start writing at. + * @throws IOException On I/O error. + */ + public static void writeFully(WritableByteChannel bc, ByteBuffer buf) + throws IOException { + do { + bc.write(buf); + } while (buf.remaining() > 0); + } + + /** + * Write a ByteBuffer to a FileChannel at a given offset, + * handling short writes. + * + * @param fc The FileChannel to write to. + * @param buf The input buffer + * @param offset The offset in the file to start writing at. + * @throws IOException On I/O error. + */ + public static void writeFully(FileChannel fc, ByteBuffer buf, + long offset) throws IOException { + do { + offset += fc.write(buf, offset); + } while (buf.remaining() > 0); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java index d4f5057f7c..60c0703abc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java @@ -21,9 +21,13 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import org.junit.Test; import org.mockito.Mockito; @@ -32,7 +36,8 @@ * Test cases for IOUtils.java */ public class TestIOUtils { - + private static final String TEST_FILE_NAME = "test_file"; + @Test public void testCopyBytesShouldCloseStreamsWhenCloseIsTrue() throws Exception { InputStream inputStream = Mockito.mock(InputStream.class); @@ -110,4 +115,41 @@ public void testCopyBytesWithCountShouldThrowOutTheStreamClosureExceptions() Mockito.verify(outputStream, Mockito.atLeastOnce()).close(); } + @Test + public void testWriteFully() throws IOException { + final int INPUT_BUFFER_LEN = 10000; + final int HALFWAY = 1 + (INPUT_BUFFER_LEN / 2); + byte[] input = new byte[INPUT_BUFFER_LEN]; + for (int i = 0; i < input.length; i++) { + input[i] = (byte)(i & 0xff); + } + byte[] output = new byte[input.length]; + + try { + RandomAccessFile raf = new RandomAccessFile(TEST_FILE_NAME, "rw"); + FileChannel fc = raf.getChannel(); + ByteBuffer buf = ByteBuffer.wrap(input); + IOUtils.writeFully(fc, buf); + raf.seek(0); + raf.read(output); + for (int i = 0; i < input.length; i++) { + assertEquals(input[i], output[i]); + } + buf.rewind(); + IOUtils.writeFully(fc, buf, HALFWAY); + for (int i = 0; i < HALFWAY; i++) { + assertEquals(input[i], output[i]); + } + raf.seek(0); + raf.read(output); + for (int i = HALFWAY; i < input.length; i++) { + assertEquals(input[i - HALFWAY], output[i]); + } + } finally { + File f = new File(TEST_FILE_NAME); + if (f.exists()) { + f.delete(); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java index dd8102ee74..08a560ce12 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java @@ -206,10 +206,10 @@ private void preallocate() throws IOException { + fc.size()); } fill.position(0); - int written = fc.write(fill, position); + IOUtils.writeFully(fc, fill, position); if(FSNamesystem.LOG.isDebugEnabled()) { FSNamesystem.LOG.debug("Edit log size is now " + fc.size() + - " written " + written + " bytes " + " at offset " + position); + " written " + fill.capacity() + " bytes " + " at offset " + position); } } } From 009c58f2af36e8e9d50311a00b5a4699b71463d3 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Mon, 11 Jun 2012 23:14:06 +0000 Subject: [PATCH 26/91] MAPREDUCE-3921. MR AM should act on node health status changes. Contributed by Bikas Saha. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349065 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapreduce/v2/app/job/TaskAttempt.java | 6 + .../v2/app/job/event/JobEventType.java | 6 +- .../app/job/event/JobUpdatedNodesEvent.java | 40 ++++ .../app/job/event/TaskAttemptKillEvent.java | 37 ++++ .../mapreduce/v2/app/job/impl/JobImpl.java | 106 ++++++++-- .../v2/app/job/impl/TaskAttemptImpl.java | 78 ++++++-- .../mapreduce/v2/app/job/impl/TaskImpl.java | 67 +++++-- .../v2/app/rm/RMContainerAllocator.java | 92 +++++++-- .../hadoop/mapreduce/v2/app/MockJobs.java | 6 + .../hadoop/mapreduce/v2/app/TestMRApp.java | 182 +++++++++++++++++- .../v2/app/TestRMContainerAllocator.java | 101 +++++++++- .../v2/app/TestRuntimeEstimators.java | 6 + .../jobhistory/JobHistoryParser.java | 7 +- .../mapreduce/v2/hs/CompletedTaskAttempt.java | 6 + .../hadoop/yarn/api/records/NodeState.java | 8 +- 16 files changed, 686 insertions(+), 65 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobUpdatedNodesEvent.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptKillEvent.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 7c7897425b..ce0e8e031c 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -132,6 +132,9 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-3871. Allow symlinking in LocalJobRunner DistributedCache. (tomwhite) + MAPREDUCE-3921. MR AM should act on node health status changes. + (Bikas Saha via sseth) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java index e627128975..af4aef7d9c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java @@ -25,6 +25,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; /** @@ -54,6 +55,11 @@ public interface TaskAttempt { */ String getAssignedContainerMgrAddress(); + /** + * @return node's id if a container is assigned, otherwise null. + */ + NodeId getNodeId(); + /** * @return node's http address if a container is assigned, otherwise null. */ diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobEventType.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobEventType.java index e0223b1379..5accb6da3e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobEventType.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobEventType.java @@ -44,5 +44,9 @@ public enum JobEventType { JOB_COUNTER_UPDATE, //Producer:TaskAttemptListener - JOB_TASK_ATTEMPT_FETCH_FAILURE + JOB_TASK_ATTEMPT_FETCH_FAILURE, + + //Producer:RMContainerAllocator + JOB_UPDATED_NODES + } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobUpdatedNodesEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobUpdatedNodesEvent.java new file mode 100644 index 0000000000..c332fb0e69 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobUpdatedNodesEvent.java @@ -0,0 +1,40 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.mapreduce.v2.app.job.event; + +import java.util.List; + +import org.apache.hadoop.mapreduce.v2.api.records.JobId; +import org.apache.hadoop.yarn.api.records.NodeReport; + + + +public class JobUpdatedNodesEvent extends JobEvent { + + private final List updatedNodes; + public JobUpdatedNodesEvent(JobId jobId, List updatedNodes) { + super(jobId, JobEventType.JOB_UPDATED_NODES); + this.updatedNodes = updatedNodes; + } + + public List getUpdatedNodes() { + return updatedNodes; + } + +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptKillEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptKillEvent.java new file mode 100644 index 0000000000..9bcc838173 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptKillEvent.java @@ -0,0 +1,37 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.mapreduce.v2.app.job.event; + +import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; + + +public class TaskAttemptKillEvent extends TaskAttemptEvent { + + private final String message; + + public TaskAttemptKillEvent(TaskAttemptId attemptID, + String message) { + super(attemptID, TaskAttemptEventType.TA_KILL); + this.message = message; + } + + public String getMessage() { + return message; + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index 220bbc31d6..10eb68dbf8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -77,6 +77,7 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener; import org.apache.hadoop.mapreduce.v2.app.job.Task; +import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent; @@ -85,8 +86,10 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptCompletedEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptFetchFailureEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskEvent; +import org.apache.hadoop.mapreduce.v2.app.job.event.JobUpdatedNodesEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType; import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics; @@ -100,6 +103,9 @@ import org.apache.hadoop.yarn.Clock; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.state.InvalidStateTransitonException; import org.apache.hadoop.yarn.state.MultipleArcTransition; @@ -148,6 +154,12 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, private final Object tasksSyncHandle = new Object(); private final Set mapTasks = new LinkedHashSet(); private final Set reduceTasks = new LinkedHashSet(); + /** + * maps nodes to tasks that have run on those nodes + */ + private final HashMap> + nodesToSucceededTaskAttempts = new HashMap>(); + private final EventHandler eventHandler; private final MRAppMetrics metrics; private final String userName; @@ -194,6 +206,8 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, new TaskAttemptCompletedEventTransition(); private static final CounterUpdateTransition COUNTER_UPDATE_TRANSITION = new CounterUpdateTransition(); + private static final UpdatedNodesTransition UPDATED_NODES_TRANSITION = + new UpdatedNodesTransition(); protected static final StateMachineFactory @@ -218,7 +232,10 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, .addTransition(JobState.NEW, JobState.ERROR, JobEventType.INTERNAL_ERROR, INTERNAL_ERROR_TRANSITION) - + // Ignore-able events + .addTransition(JobState.NEW, JobState.NEW, + JobEventType.JOB_UPDATED_NODES) + // Transitions from INITED state .addTransition(JobState.INITED, JobState.INITED, JobEventType.JOB_DIAGNOSTIC_UPDATE, @@ -234,7 +251,10 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, .addTransition(JobState.INITED, JobState.ERROR, JobEventType.INTERNAL_ERROR, INTERNAL_ERROR_TRANSITION) - + // Ignore-able events + .addTransition(JobState.INITED, JobState.INITED, + JobEventType.JOB_UPDATED_NODES) + // Transitions from RUNNING state .addTransition(JobState.RUNNING, JobState.RUNNING, JobEventType.JOB_TASK_ATTEMPT_COMPLETED, @@ -251,6 +271,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, new JobNoTasksCompletedTransition()) .addTransition(JobState.RUNNING, JobState.KILL_WAIT, JobEventType.JOB_KILL, new KillTasksTransition()) + .addTransition(JobState.RUNNING, JobState.RUNNING, + JobEventType.JOB_UPDATED_NODES, + UPDATED_NODES_TRANSITION) .addTransition(JobState.RUNNING, JobState.RUNNING, JobEventType.JOB_MAP_TASK_RESCHEDULED, new MapTaskRescheduledTransition()) @@ -288,8 +311,9 @@ JobEventType.JOB_KILL, new KillTasksTransition()) // Ignore-able events .addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT, EnumSet.of(JobEventType.JOB_KILL, - JobEventType.JOB_MAP_TASK_RESCHEDULED, - JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE)) + JobEventType.JOB_UPDATED_NODES, + JobEventType.JOB_MAP_TASK_RESCHEDULED, + JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE)) // Transitions from SUCCEEDED state .addTransition(JobState.SUCCEEDED, JobState.SUCCEEDED, @@ -303,7 +327,8 @@ JobEventType.JOB_KILL, new KillTasksTransition()) INTERNAL_ERROR_TRANSITION) // Ignore-able events .addTransition(JobState.SUCCEEDED, JobState.SUCCEEDED, - EnumSet.of(JobEventType.JOB_KILL, + EnumSet.of(JobEventType.JOB_KILL, + JobEventType.JOB_UPDATED_NODES, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE)) // Transitions from FAILED state @@ -318,7 +343,8 @@ JobEventType.JOB_KILL, new KillTasksTransition()) INTERNAL_ERROR_TRANSITION) // Ignore-able events .addTransition(JobState.FAILED, JobState.FAILED, - EnumSet.of(JobEventType.JOB_KILL, + EnumSet.of(JobEventType.JOB_KILL, + JobEventType.JOB_UPDATED_NODES, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE)) // Transitions from KILLED state @@ -333,7 +359,8 @@ JobEventType.JOB_KILL, new KillTasksTransition()) INTERNAL_ERROR_TRANSITION) // Ignore-able events .addTransition(JobState.KILLED, JobState.KILLED, - EnumSet.of(JobEventType.JOB_KILL, + EnumSet.of(JobEventType.JOB_KILL, + JobEventType.JOB_UPDATED_NODES, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE)) // No transitions from INTERNAL_ERROR state. Ignore all. @@ -346,6 +373,7 @@ JobEventType.JOB_KILL, new KillTasksTransition()) JobEventType.JOB_TASK_ATTEMPT_COMPLETED, JobEventType.JOB_MAP_TASK_RESCHEDULED, JobEventType.JOB_DIAGNOSTIC_UPDATE, + JobEventType.JOB_UPDATED_NODES, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE, JobEventType.INTERNAL_ERROR)) .addTransition(JobState.ERROR, JobState.ERROR, @@ -895,7 +923,7 @@ private void makeUberDecision(long dataInputLength) { LOG.info(msg.toString()); } } - + /** * ChainMapper and ChainReducer must execute in parallel, so they're not * compatible with uberization/LocalContainerLauncher (100% sequential). @@ -924,6 +952,24 @@ private boolean isChainJob(Configuration conf) { } return isChainJob; } + + private void actOnUnusableNode(NodeId nodeId, NodeState nodeState) { + // rerun previously successful map tasks + List taskAttemptIdList = nodesToSucceededTaskAttempts.get(nodeId); + if(taskAttemptIdList != null) { + String mesg = "TaskAttempt killed because it ran on unusable node " + + nodeId; + for(TaskAttemptId id : taskAttemptIdList) { + if(TaskType.MAP == id.getTaskId().getTaskType()) { + // reschedule only map tasks because their outputs maybe unusable + LOG.info(mesg + ". AttemptId:" + id); + eventHandler.handle(new TaskAttemptKillEvent(id, mesg)); + } + } + } + // currently running task attempts on unusable nodes are handled in + // RMContainerAllocator + } /* private int getBlockSize() { @@ -1269,18 +1315,37 @@ public void transition(JobImpl job, JobEvent event) { tce.setEventId(job.taskAttemptCompletionEvents.size()); job.taskAttemptCompletionEvents.add(tce); + TaskAttemptId attemptId = tce.getAttemptId(); + TaskId taskId = attemptId.getTaskId(); //make the previous completion event as obsolete if it exists Object successEventNo = - job.successAttemptCompletionEventNoMap.remove(tce.getAttemptId().getTaskId()); + job.successAttemptCompletionEventNoMap.remove(taskId); if (successEventNo != null) { TaskAttemptCompletionEvent successEvent = job.taskAttemptCompletionEvents.get((Integer) successEventNo); successEvent.setStatus(TaskAttemptCompletionEventStatus.OBSOLETE); } - + + // if this attempt is not successful then why is the previous successful + // attempt being removed above - MAPREDUCE-4330 if (TaskAttemptCompletionEventStatus.SUCCEEDED.equals(tce.getStatus())) { - job.successAttemptCompletionEventNoMap.put(tce.getAttemptId().getTaskId(), - tce.getEventId()); + job.successAttemptCompletionEventNoMap.put(taskId, tce.getEventId()); + + // here we could have simply called Task.getSuccessfulAttempt() but + // the event that triggers this code is sent before + // Task.successfulAttempt is set and so there is no guarantee that it + // will be available now + Task task = job.tasks.get(taskId); + TaskAttempt attempt = task.getAttempt(attemptId); + NodeId nodeId = attempt.getNodeId(); + assert (nodeId != null); // node must exist for a successful event + List taskAttemptIdList = job.nodesToSucceededTaskAttempts + .get(nodeId); + if (taskAttemptIdList == null) { + taskAttemptIdList = new ArrayList(); + job.nodesToSucceededTaskAttempts.put(nodeId, taskAttemptIdList); + } + taskAttemptIdList.add(attempt.getID()); } } } @@ -1460,7 +1525,22 @@ public void transition(JobImpl job, JobEvent event) { } } } - + + private static class UpdatedNodesTransition implements + SingleArcTransition { + @Override + public void transition(JobImpl job, JobEvent event) { + JobUpdatedNodesEvent updateEvent = (JobUpdatedNodesEvent) event; + for(NodeReport nr: updateEvent.getUpdatedNodes()) { + NodeState nodeState = nr.getNodeState(); + if(nodeState.isUnusable()) { + // act on the updates + job.actOnUnusableNode(nr.getNodeId(), nodeState); + } + } + } + } + private static class InternalErrorTransition implements SingleArcTransition { @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index cafff92920..66d48b6904 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -84,6 +84,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType; @@ -403,6 +404,10 @@ TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition()) TaskAttemptState.FAILED, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE, new TooManyFetchFailureTransition()) + .addTransition( + TaskAttemptState.SUCCEEDED, TaskAttemptState.KILLED, + TaskAttemptEventType.TA_KILL, + new KilledAfterSuccessTransition()) .addTransition( TaskAttemptState.SUCCEEDED, TaskAttemptState.SUCCEEDED, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, @@ -410,8 +415,7 @@ TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition()) // Ignore-able events for SUCCEEDED state .addTransition(TaskAttemptState.SUCCEEDED, TaskAttemptState.SUCCEEDED, - EnumSet.of(TaskAttemptEventType.TA_KILL, - TaskAttemptEventType.TA_FAILMSG, + EnumSet.of(TaskAttemptEventType.TA_FAILMSG, TaskAttemptEventType.TA_CONTAINER_CLEANED, TaskAttemptEventType.TA_CONTAINER_COMPLETED)) @@ -818,6 +822,16 @@ public int getShufflePort() { } } + @Override + public NodeId getNodeId() { + readLock.lock(); + try { + return containerNodeId; + } finally { + readLock.unlock(); + } + } + /**If container Assigned then return the node's address, otherwise null. */ @Override @@ -999,7 +1013,7 @@ private static long computeSlotMillis(TaskAttemptImpl taskAttempt) { } private static JobCounterUpdateEvent createJobCounterUpdateEventTAFailed( - TaskAttemptImpl taskAttempt) { + TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) { TaskType taskType = taskAttempt.getID().getTaskId().getTaskType(); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId()); @@ -1007,16 +1021,22 @@ private static JobCounterUpdateEvent createJobCounterUpdateEventTAFailed( if (taskType == TaskType.MAP) { jce.addCounterUpdate(JobCounter.NUM_FAILED_MAPS, 1); - jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, slotMillisIncrement); + if(!taskAlreadyCompleted) { + // dont double count the elapsed time + jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, slotMillisIncrement); + } } else { jce.addCounterUpdate(JobCounter.NUM_FAILED_REDUCES, 1); - jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, slotMillisIncrement); + if(!taskAlreadyCompleted) { + // dont double count the elapsed time + jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, slotMillisIncrement); + } } return jce; } private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled( - TaskAttemptImpl taskAttempt) { + TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) { TaskType taskType = taskAttempt.getID().getTaskId().getTaskType(); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId()); @@ -1024,10 +1044,16 @@ private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled( if (taskType == TaskType.MAP) { jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1); - jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, slotMillisIncrement); + if(!taskAlreadyCompleted) { + // dont double count the elapsed time + jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, slotMillisIncrement); + } } else { jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1); - jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, slotMillisIncrement); + if(!taskAlreadyCompleted) { + // dont double count the elapsed time + jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, slotMillisIncrement); + } } return jce; } @@ -1259,10 +1285,10 @@ public void transition(TaskAttemptImpl taskAttempt, finalState); if(finalState == TaskAttemptState.FAILED) { taskAttempt.eventHandler - .handle(createJobCounterUpdateEventTAFailed(taskAttempt)); + .handle(createJobCounterUpdateEventTAFailed(taskAttempt, false)); } else if(finalState == TaskAttemptState.KILLED) { taskAttempt.eventHandler - .handle(createJobCounterUpdateEventTAKilled(taskAttempt)); + .handle(createJobCounterUpdateEventTAKilled(taskAttempt, false)); } taskAttempt.eventHandler.handle(new JobHistoryEvent( taskAttempt.attemptId.getTaskId().getJobId(), tauce)); @@ -1394,7 +1420,7 @@ public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) { if (taskAttempt.getLaunchTime() != 0) { taskAttempt.eventHandler - .handle(createJobCounterUpdateEventTAFailed(taskAttempt)); + .handle(createJobCounterUpdateEventTAFailed(taskAttempt, false)); TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt, TaskAttemptState.FAILED); @@ -1463,7 +1489,7 @@ public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) { if (taskAttempt.getLaunchTime() != 0) { taskAttempt.eventHandler - .handle(createJobCounterUpdateEventTAFailed(taskAttempt)); + .handle(createJobCounterUpdateEventTAFailed(taskAttempt, true)); TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt, TaskAttemptState.FAILED); @@ -1477,6 +1503,32 @@ public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) { taskAttempt.attemptId, TaskEventType.T_ATTEMPT_FAILED)); } } + + private static class KilledAfterSuccessTransition implements + SingleArcTransition { + + @SuppressWarnings("unchecked") + @Override + public void transition(TaskAttemptImpl taskAttempt, + TaskAttemptEvent event) { + TaskAttemptKillEvent msgEvent = (TaskAttemptKillEvent) event; + //add to diagnostic + taskAttempt.addDiagnosticInfo(msgEvent.getMessage()); + + // not setting a finish time since it was set on success + assert (taskAttempt.getFinishTime() != 0); + + assert (taskAttempt.getLaunchTime() != 0); + taskAttempt.eventHandler + .handle(createJobCounterUpdateEventTAKilled(taskAttempt, true)); + TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent( + taskAttempt, TaskAttemptState.KILLED); + taskAttempt.eventHandler.handle(new JobHistoryEvent(taskAttempt.attemptId + .getTaskId().getJobId(), tauce)); + taskAttempt.eventHandler.handle(new TaskTAttemptEvent( + taskAttempt.attemptId, TaskEventType.T_ATTEMPT_KILLED)); + } + } private static class KilledTransition implements SingleArcTransition { @@ -1489,7 +1541,7 @@ public void transition(TaskAttemptImpl taskAttempt, taskAttempt.setFinishTime(); if (taskAttempt.getLaunchTime() != 0) { taskAttempt.eventHandler - .handle(createJobCounterUpdateEventTAKilled(taskAttempt)); + .handle(createJobCounterUpdateEventTAKilled(taskAttempt, false)); TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt, TaskAttemptState.KILLED); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java index 58edd1690c..c7f89f9ac5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java @@ -191,13 +191,14 @@ TaskEventType.T_ADD_SPEC_ATTEMPT, new RedundantScheduleTransition()) .addTransition(TaskState.SUCCEEDED, //only possible for map tasks EnumSet.of(TaskState.SCHEDULED, TaskState.FAILED), TaskEventType.T_ATTEMPT_FAILED, new MapRetroactiveFailureTransition()) + .addTransition(TaskState.SUCCEEDED, //only possible for map tasks + EnumSet.of(TaskState.SCHEDULED, TaskState.SUCCEEDED), + TaskEventType.T_ATTEMPT_KILLED, new MapRetroactiveKilledTransition()) // Ignore-able transitions. .addTransition( TaskState.SUCCEEDED, TaskState.SUCCEEDED, - EnumSet.of(TaskEventType.T_KILL, - TaskEventType.T_ADD_SPEC_ATTEMPT, - TaskEventType.T_ATTEMPT_LAUNCHED, - TaskEventType.T_ATTEMPT_KILLED)) + EnumSet.of(TaskEventType.T_ADD_SPEC_ATTEMPT, + TaskEventType.T_ATTEMPT_LAUNCHED)) // Transitions from FAILED state .addTransition(TaskState.FAILED, TaskState.FAILED, @@ -629,7 +630,6 @@ private void internalError(TaskEventType type) { // always called inside a transition, in turn inside the Write Lock private void handleTaskAttemptCompletion(TaskAttemptId attemptId, TaskAttemptCompletionEventStatus status) { - finishedAttempts++; TaskAttempt attempt = attempts.get(attemptId); //raise the completion event only if the container is assigned // to nextAttemptNumber @@ -681,6 +681,11 @@ private static TaskFailedEvent createTaskFailedEvent(TaskImpl task, List taId == null ? null : TypeConverter.fromYarn(taId)); return taskFailedEvent; } + + private static void unSucceed(TaskImpl task) { + task.commitAttempt = null; + task.successfulAttempt = null; + } /** * @return a String representation of the splits. @@ -755,6 +760,7 @@ public void transition(TaskImpl task, TaskEvent event) { task.handleTaskAttemptCompletion( ((TaskTAttemptEvent) event).getTaskAttemptID(), TaskAttemptCompletionEventStatus.SUCCEEDED); + task.finishedAttempts++; --task.numberUncompletedAttempts; task.successfulAttempt = ((TaskTAttemptEvent) event).getTaskAttemptID(); task.eventHandler.handle(new JobTaskEvent( @@ -790,6 +796,7 @@ public void transition(TaskImpl task, TaskEvent event) { task.handleTaskAttemptCompletion( ((TaskTAttemptEvent) event).getTaskAttemptID(), TaskAttemptCompletionEventStatus.KILLED); + task.finishedAttempts++; --task.numberUncompletedAttempts; if (task.successfulAttempt == null) { task.addAndScheduleAttempt(); @@ -808,6 +815,7 @@ public TaskState transition(TaskImpl task, TaskEvent event) { task.handleTaskAttemptCompletion( ((TaskTAttemptEvent) event).getTaskAttemptID(), TaskAttemptCompletionEventStatus.KILLED); + task.finishedAttempts++; // check whether all attempts are finished if (task.finishedAttempts == task.attempts.size()) { if (task.historyTaskStartGenerated) { @@ -845,6 +853,7 @@ public TaskState transition(TaskImpl task, TaskEvent event) { attempt.getAssignedContainerMgrAddress())); } + task.finishedAttempts++; if (task.failedAttempts < task.maxAttempts) { task.handleTaskAttemptCompletion( ((TaskTAttemptEvent) event).getTaskAttemptID(), @@ -880,12 +889,6 @@ public TaskState transition(TaskImpl task, TaskEvent event) { protected TaskState getDefaultState(Task task) { return task.getState(); } - - protected void unSucceed(TaskImpl task) { - ++task.numberUncompletedAttempts; - task.commitAttempt = null; - task.successfulAttempt = null; - } } private static class MapRetroactiveFailureTransition @@ -908,6 +911,8 @@ public TaskState transition(TaskImpl task, TaskEvent event) { // fails, we have to let AttemptFailedTransition.transition // believe that there's no redundancy. unSucceed(task); + // fake increase in Uncomplete attempts for super.transition + ++task.numberUncompletedAttempts; return super.transition(task, event); } @@ -917,6 +922,45 @@ protected TaskState getDefaultState(Task task) { } } + private static class MapRetroactiveKilledTransition implements + MultipleArcTransition { + + @Override + public TaskState transition(TaskImpl task, TaskEvent event) { + // verify that this occurs only for map task + // TODO: consider moving it to MapTaskImpl + if (!TaskType.MAP.equals(task.getType())) { + LOG.error("Unexpected event for REDUCE task " + event.getType()); + task.internalError(event.getType()); + } + + TaskTAttemptEvent attemptEvent = (TaskTAttemptEvent) event; + TaskAttemptId attemptId = attemptEvent.getTaskAttemptID(); + if(task.successfulAttempt == attemptId) { + // successful attempt is now killed. reschedule + // tell the job about the rescheduling + unSucceed(task); + task.handleTaskAttemptCompletion( + attemptId, + TaskAttemptCompletionEventStatus.KILLED); + task.eventHandler.handle(new JobMapTaskRescheduledEvent(task.taskId)); + // typically we are here because this map task was run on a bad node and + // we want to reschedule it on a different node. + // Depending on whether there are previous failed attempts or not this + // can SCHEDULE or RESCHEDULE the container allocate request. If this + // SCHEDULE's then the dataLocal hosts of this taskAttempt will be used + // from the map splitInfo. So the bad node might be sent as a location + // to the RM. But the RM would ignore that just like it would ignore + // currently pending container requests affinitized to bad nodes. + task.addAndScheduleAttempt(); + return TaskState.SCHEDULED; + } else { + // nothing to do + return TaskState.SUCCEEDED; + } + } + } + private static class KillNewTransition implements SingleArcTransition { @Override @@ -966,6 +1010,7 @@ static class LaunchTransition public void transition(TaskImpl task, TaskEvent event) { task.metrics.launchedTask(task); task.metrics.runningTask(task); + } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index bcb82230d6..efef456f10 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -46,19 +47,27 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.client.ClientService; +import org.apache.hadoop.mapreduce.v2.app.job.Job; +import org.apache.hadoop.mapreduce.v2.app.job.Task; +import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType; +import org.apache.hadoop.mapreduce.v2.app.job.event.JobUpdatedNodesEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.AMResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.util.RackResolver; @@ -583,7 +592,9 @@ private List getResources() throws Exception { //Called on each allocation. Will know about newly blacklisted/added hosts. computeIgnoreBlacklisting(); - + + handleUpdatedNodes(response); + for (ContainerStatus cont : finishedContainers) { LOG.info("Received completed container " + cont.getContainerId()); TaskAttemptId attemptID = assignedRequests.get(cont.getContainerId()); @@ -600,10 +611,48 @@ private List getResources() throws Exception { String diagnostics = cont.getDiagnostics(); eventHandler.handle(new TaskAttemptDiagnosticsUpdateEvent(attemptID, diagnostics)); - } + } } return newContainers; } + + @SuppressWarnings("unchecked") + private void handleUpdatedNodes(AMResponse response) { + // send event to the job about on updated nodes + List updatedNodes = response.getUpdatedNodes(); + if (!updatedNodes.isEmpty()) { + + // send event to the job to act upon completed tasks + eventHandler.handle(new JobUpdatedNodesEvent(getJob().getID(), + updatedNodes)); + + // act upon running tasks + HashSet unusableNodes = new HashSet(); + for (NodeReport nr : updatedNodes) { + NodeState nodeState = nr.getNodeState(); + if (nodeState.isUnusable()) { + unusableNodes.add(nr.getNodeId()); + } + } + for (int i = 0; i < 2; ++i) { + HashMap taskSet = i == 0 ? assignedRequests.maps + : assignedRequests.reduces; + // kill running containers + for (Map.Entry entry : taskSet.entrySet()) { + TaskAttemptId tid = entry.getKey(); + NodeId taskAttemptNodeId = entry.getValue().getNodeId(); + if (unusableNodes.contains(taskAttemptNodeId)) { + LOG.info("Killing taskAttempt:" + tid + + " because it is running on unusable node:" + + taskAttemptNodeId); + eventHandler.handle(new TaskAttemptKillEvent(tid, + "TaskAttempt killed because it ran on unusable node" + + taskAttemptNodeId)); + } + } + } + } + } @Private public int getMemLimit() { @@ -743,7 +792,6 @@ else if (PRIORITY_REDUCE.equals(priority)) { boolean blackListed = false; ContainerRequest assigned = null; - ContainerId allocatedContainerId = allocated.getId(); if (isAssignable) { // do not assign if allocated container is on a // blacklisted host @@ -790,7 +838,7 @@ else if (PRIORITY_REDUCE.equals(priority)) { eventHandler.handle(new TaskAttemptContainerAssignedEvent( assigned.attemptID, allocated, applicationACLs)); - assignedRequests.add(allocatedContainerId, assigned.attemptID); + assignedRequests.add(allocated, assigned.attemptID); if (LOG.isDebugEnabled()) { LOG.info("Assigned container (" + allocated + ") " @@ -811,7 +859,7 @@ else if (PRIORITY_REDUCE.equals(priority)) { // or if we could not assign it if (blackListed || assigned == null) { containersReleased++; - release(allocatedContainerId); + release(allocated.getId()); } } } @@ -974,20 +1022,20 @@ private ContainerRequest assignToMap(Container allocated) { private class AssignedRequests { private final Map containerToAttemptMap = new HashMap(); - private final LinkedHashMap maps = - new LinkedHashMap(); - private final LinkedHashMap reduces = - new LinkedHashMap(); + private final LinkedHashMap maps = + new LinkedHashMap(); + private final LinkedHashMap reduces = + new LinkedHashMap(); private final Set preemptionWaitingReduces = new HashSet(); - void add(ContainerId containerId, TaskAttemptId tId) { - LOG.info("Assigned container " + containerId.toString() + " to " + tId); - containerToAttemptMap.put(containerId, tId); + void add(Container container, TaskAttemptId tId) { + LOG.info("Assigned container " + container.getId().toString() + " to " + tId); + containerToAttemptMap.put(container.getId(), tId); if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) { - maps.put(tId, containerId); + maps.put(tId, container); } else { - reduces.put(tId, containerId); + reduces.put(tId, container); } } @@ -1017,9 +1065,9 @@ public int compare(TaskAttemptId o1, TaskAttemptId o2) { boolean remove(TaskAttemptId tId) { ContainerId containerId = null; if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) { - containerId = maps.remove(tId); + containerId = maps.remove(tId).getId(); } else { - containerId = reduces.remove(tId); + containerId = reduces.remove(tId).getId(); if (containerId != null) { boolean preempted = preemptionWaitingReduces.remove(tId); if (preempted) { @@ -1038,12 +1086,20 @@ boolean remove(TaskAttemptId tId) { TaskAttemptId get(ContainerId cId) { return containerToAttemptMap.get(cId); } + + NodeId getNodeId(TaskAttemptId tId) { + if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) { + return maps.get(tId).getNodeId(); + } else { + return reduces.get(tId).getNodeId(); + } + } ContainerId get(TaskAttemptId tId) { if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) { - return maps.get(tId); + return maps.get(tId).getId(); } else { - return reduces.get(tId); + return reduces.get(tId).getId(); } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java index 81af358bc2..7f1d820032 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java @@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.Records; @@ -231,6 +232,11 @@ public static TaskAttempt newTaskAttempt(TaskId tid, int i) { final List diags = Lists.newArrayList(); diags.add(DIAGS.next()); return new TaskAttempt() { + @Override + public NodeId getNodeId() throws UnsupportedOperationException{ + throw new UnsupportedOperationException(); + } + @Override public TaskAttemptId getID() { return taid; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java index 3ca9c24bad..a0d8e77824 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import java.util.ArrayList; import java.util.Iterator; import junit.framework.Assert; @@ -29,17 +30,26 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TypeConverter; +import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent; +import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler; import org.apache.hadoop.mapreduce.v2.api.records.JobState; +import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState; import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; +import org.apache.hadoop.mapreduce.v2.app.job.event.JobUpdatedNodesEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType; import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.junit.Test; /** @@ -160,6 +170,159 @@ public void testCompletedMapsForReduceSlowstart() throws Exception { app.waitForState(job, JobState.SUCCEEDED); } + + /** + * The test verifies that the AM re-runs maps that have run on bad nodes. It + * also verifies that the AM records all success/killed events so that reduces + * are notified about map output status changes. It also verifies that the + * re-run information is preserved across AM restart + */ + @Test + public void testUpdatedNodes() throws Exception { + int runCount = 0; + MRApp app = new MRAppWithHistory(2, 1, false, this.getClass().getName(), + true, ++runCount); + Configuration conf = new Configuration(); + // after half of the map completion, reduce will start + conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.5f); + // uberization forces full slowstart (1.0), so disable that + conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); + Job job = app.submit(conf); + app.waitForState(job, JobState.RUNNING); + Assert.assertEquals("Num tasks not correct", 3, job.getTasks().size()); + Iterator it = job.getTasks().values().iterator(); + Task mapTask1 = it.next(); + Task mapTask2 = it.next(); + + // all maps must be running + app.waitForState(mapTask1, TaskState.RUNNING); + app.waitForState(mapTask2, TaskState.RUNNING); + + TaskAttempt task1Attempt = mapTask1.getAttempts().values().iterator() + .next(); + TaskAttempt task2Attempt = mapTask2.getAttempts().values().iterator() + .next(); + NodeId node1 = task1Attempt.getNodeId(); + NodeId node2 = task2Attempt.getNodeId(); + Assert.assertEquals(node1, node2); + + // send the done signal to the task + app.getContext() + .getEventHandler() + .handle( + new TaskAttemptEvent(task1Attempt.getID(), + TaskAttemptEventType.TA_DONE)); + app.getContext() + .getEventHandler() + .handle( + new TaskAttemptEvent(task2Attempt.getID(), + TaskAttemptEventType.TA_DONE)); + + // all maps must be succeeded + app.waitForState(mapTask1, TaskState.SUCCEEDED); + app.waitForState(mapTask2, TaskState.SUCCEEDED); + + TaskAttemptCompletionEvent[] events = job.getTaskAttemptCompletionEvents(0, + 100); + Assert.assertEquals("Expecting 2 completion events for success", 2, + events.length); + + // send updated nodes info + ArrayList updatedNodes = new ArrayList(); + NodeReport nr = RecordFactoryProvider.getRecordFactory(null) + .newRecordInstance(NodeReport.class); + nr.setNodeId(node1); + nr.setNodeState(NodeState.UNHEALTHY); + updatedNodes.add(nr); + app.getContext().getEventHandler() + .handle(new JobUpdatedNodesEvent(job.getID(), updatedNodes)); + + app.waitForState(task1Attempt, TaskAttemptState.KILLED); + app.waitForState(task2Attempt, TaskAttemptState.KILLED); + + events = job.getTaskAttemptCompletionEvents(0, 100); + Assert.assertEquals("Expecting 2 more completion events for killed", 4, + events.length); + + // all maps must be back to running + app.waitForState(mapTask1, TaskState.RUNNING); + app.waitForState(mapTask2, TaskState.RUNNING); + + Iterator itr = mapTask1.getAttempts().values().iterator(); + itr.next(); + task1Attempt = itr.next(); + + // send the done signal to the task + app.getContext() + .getEventHandler() + .handle( + new TaskAttemptEvent(task1Attempt.getID(), + TaskAttemptEventType.TA_DONE)); + + // map1 must be succeeded. map2 must be running + app.waitForState(mapTask1, TaskState.SUCCEEDED); + app.waitForState(mapTask2, TaskState.RUNNING); + + events = job.getTaskAttemptCompletionEvents(0, 100); + Assert.assertEquals("Expecting 1 more completion events for success", 5, + events.length); + + // Crash the app again. + app.stop(); + + // rerun + // in rerun the 1st map will be recovered from previous run + app = new MRAppWithHistory(2, 1, false, this.getClass().getName(), false, + ++runCount); + conf = new Configuration(); + conf.setBoolean(MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE, true); + conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); + job = app.submit(conf); + app.waitForState(job, JobState.RUNNING); + Assert.assertEquals("No of tasks not correct", 3, job.getTasks().size()); + it = job.getTasks().values().iterator(); + mapTask1 = it.next(); + mapTask2 = it.next(); + Task reduceTask = it.next(); + + // map 1 will be recovered, no need to send done + app.waitForState(mapTask1, TaskState.SUCCEEDED); + app.waitForState(mapTask2, TaskState.RUNNING); + + events = job.getTaskAttemptCompletionEvents(0, 100); + Assert.assertEquals( + "Expecting 2 completion events for killed & success of map1", 2, + events.length); + + task2Attempt = mapTask2.getAttempts().values().iterator().next(); + app.getContext() + .getEventHandler() + .handle( + new TaskAttemptEvent(task2Attempt.getID(), + TaskAttemptEventType.TA_DONE)); + app.waitForState(mapTask2, TaskState.SUCCEEDED); + + events = job.getTaskAttemptCompletionEvents(0, 100); + Assert.assertEquals("Expecting 1 more completion events for success", 3, + events.length); + + app.waitForState(reduceTask, TaskState.RUNNING); + TaskAttempt task3Attempt = reduceTask.getAttempts().values().iterator() + .next(); + app.getContext() + .getEventHandler() + .handle( + new TaskAttemptEvent(task3Attempt.getID(), + TaskAttemptEventType.TA_DONE)); + app.waitForState(reduceTask, TaskState.SUCCEEDED); + + events = job.getTaskAttemptCompletionEvents(0, 100); + Assert.assertEquals("Expecting 1 more completion events for success", 4, + events.length); + + // job succeeds + app.waitForState(job, JobState.SUCCEEDED); + } @Test public void testJobError() throws Exception { @@ -194,10 +357,6 @@ protected Job createJob(Configuration conf) { ((AppContext) getContext()).getAllJobs().put(spiedJob.getID(), spiedJob); return spiedJob; } - - JobImpl getSpiedJob() { - return this.spiedJob; - } } @Test @@ -232,6 +391,21 @@ public void checkTaskStateTypeConversion() { TypeConverter.fromYarn(state); } } + + private final class MRAppWithHistory extends MRApp { + public MRAppWithHistory(int maps, int reduces, boolean autoComplete, + String testName, boolean cleanOnStart, int startCount) { + super(maps, reduces, autoComplete, testName, cleanOnStart, startCount); + } + + @Override + protected EventHandler createJobHistoryHandler( + AppContext context) { + JobHistoryEventHandler eventHandler = new JobHistoryEventHandler(context, + getStartCount()); + return eventHandler; + } + } public static void main(String[] args) throws Exception { TestMRApp t = new TestMRApp(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java index a7f42eed34..98bc020ecb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java @@ -46,9 +46,11 @@ import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; +import org.apache.hadoop.mapreduce.v2.app.job.event.JobUpdatedNodesEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent; import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator; import org.apache.hadoop.mapreduce.v2.app.rm.ContainerFailedEvent; import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent; @@ -594,6 +596,88 @@ protected ContainerAllocator createContainerAllocator( Assert.assertEquals(0.95f, job.getProgress(), 0.001f); Assert.assertEquals(0.95f, rmApp.getProgress(), 0.001f); } + + @Test + public void testUpdatedNodes() throws Exception { + Configuration conf = new Configuration(); + MyResourceManager rm = new MyResourceManager(conf); + rm.start(); + DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() + .getDispatcher(); + + // Submit the application + RMApp app = rm.submitApp(1024); + dispatcher.await(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + amNodeManager.nodeHeartbeat(true); + dispatcher.await(); + + ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() + .getAppAttemptId(); + rm.sendAMLaunched(appAttemptId); + dispatcher.await(); + + JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); + Job mockJob = mock(Job.class); + MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, + appAttemptId, mockJob); + + // add resources to scheduler + MockNM nm1 = rm.registerNode("h1:1234", 10240); + MockNM nm2 = rm.registerNode("h2:1234", 10240); + dispatcher.await(); + + // create the map container request + ContainerRequestEvent event = createReq(jobId, 1, 1024, + new String[] { "h1" }); + allocator.sendRequest(event); + TaskAttemptId attemptId = event.getAttemptID(); + + TaskAttempt mockTaskAttempt = mock(TaskAttempt.class); + when(mockTaskAttempt.getNodeId()).thenReturn(nm1.getNodeId()); + Task mockTask = mock(Task.class); + when(mockTask.getAttempt(attemptId)).thenReturn(mockTaskAttempt); + when(mockJob.getTask(attemptId.getTaskId())).thenReturn(mockTask); + + // this tells the scheduler about the requests + List assigned = allocator.schedule(); + dispatcher.await(); + + nm1.nodeHeartbeat(true); + dispatcher.await(); + // get the assignment + assigned = allocator.schedule(); + dispatcher.await(); + Assert.assertEquals(1, assigned.size()); + Assert.assertEquals(nm1.getNodeId(), assigned.get(0).getContainer().getNodeId()); + // no updated nodes reported + Assert.assertTrue(allocator.getJobUpdatedNodeEvents().isEmpty()); + Assert.assertTrue(allocator.getTaskAttemptKillEvents().isEmpty()); + + // mark nodes bad + nm1.nodeHeartbeat(false); + nm2.nodeHeartbeat(false); + dispatcher.await(); + + // schedule response returns updated nodes + assigned = allocator.schedule(); + dispatcher.await(); + Assert.assertEquals(0, assigned.size()); + // updated nodes are reported + Assert.assertEquals(1, allocator.getJobUpdatedNodeEvents().size()); + Assert.assertEquals(1, allocator.getTaskAttemptKillEvents().size()); + Assert.assertEquals(2, allocator.getJobUpdatedNodeEvents().get(0).getUpdatedNodes().size()); + Assert.assertEquals(attemptId, allocator.getTaskAttemptKillEvents().get(0).getTaskAttemptID()); + allocator.getJobUpdatedNodeEvents().clear(); + allocator.getTaskAttemptKillEvents().clear(); + + assigned = allocator.schedule(); + dispatcher.await(); + Assert.assertEquals(0, assigned.size()); + // no updated nodes reported + Assert.assertTrue(allocator.getJobUpdatedNodeEvents().isEmpty()); + Assert.assertTrue(allocator.getTaskAttemptKillEvents().isEmpty()); + } @Test public void testBlackListedNodes() throws Exception { @@ -1100,7 +1184,10 @@ private void checkAssignment(ContainerRequestEvent request, private static class MyContainerAllocator extends RMContainerAllocator { static final List events = new ArrayList(); - + static final List taskAttemptKillEvents + = new ArrayList(); + static final List jobUpdatedNodeEvents + = new ArrayList(); private MyResourceManager rm; private static AppContext createAppContext( @@ -1119,6 +1206,10 @@ public void handle(Event event) { // Only capture interesting events. if (event instanceof TaskAttemptContainerAssignedEvent) { events.add((TaskAttemptContainerAssignedEvent) event); + } else if (event instanceof TaskAttemptKillEvent) { + taskAttemptKillEvents.add((TaskAttemptKillEvent)event); + } else if (event instanceof JobUpdatedNodesEvent) { + jobUpdatedNodeEvents.add((JobUpdatedNodesEvent)event); } } }); @@ -1202,6 +1293,14 @@ public List schedule() { events.clear(); return result; } + + List getTaskAttemptKillEvents() { + return taskAttemptKillEvents; + } + + List getJobUpdatedNodeEvents() { + return jobUpdatedNodeEvents; + } @Override protected void startAllocatorThread() { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java index a5dae84062..80c48233c2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -528,6 +529,11 @@ void startUp() { dispatcher.getEventHandler().handle(event); } + @Override + public NodeId getNodeId() throws UnsupportedOperationException{ + throw new UnsupportedOperationException(); + } + @Override public TaskAttemptId getID() { return myAttemptID; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java index 34eb59449c..32240f888f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java @@ -282,9 +282,12 @@ private void handleTaskAttemptFailedEvent( if(attemptInfo.getAttemptId().equals(taskInfo.getSuccessfulAttemptId())) { // the failed attempt is the one that made this task successful - // so its no longer successful + // so its no longer successful. Reset fields set in + // handleTaskFinishedEvent() + taskInfo.counters = null; + taskInfo.finishTime = -1; taskInfo.status = null; - // not resetting the other fields set in handleTaskFinishedEvent() + taskInfo.successfulAttemptId = null; } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java index 84ec23e63b..8f8e77615c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java @@ -30,6 +30,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.util.Records; public class CompletedTaskAttempt implements TaskAttempt { @@ -57,6 +58,11 @@ public class CompletedTaskAttempt implements TaskAttempt { } } + @Override + public NodeId getNodeId() throws UnsupportedOperationException{ + throw new UnsupportedOperationException(); + } + @Override public ContainerId getAssignedContainerID() { return attemptInfo.getContainerId(); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeState.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeState.java index 31916c6be8..7345c258c7 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeState.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeState.java @@ -38,5 +38,9 @@ public enum NodeState { LOST, /** Node has rebooted */ - REBOOTED -} \ No newline at end of file + REBOOTED; + + public boolean isUnusable() { + return (this == UNHEALTHY || this == DECOMMISSIONED || this == LOST); + } +} From 6a560c4b653e4f74981f662c6d5b4a05778a5e63 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 12 Jun 2012 01:55:51 +0000 Subject: [PATCH 27/91] HDFS-3522. If a namenode is in safemode, it should throw SafeModeException when getBlockLocations has zero locations. Contributed by Brandon Li git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349088 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/server/namenode/FSNamesystem.java | 16 +++- .../hdfs/server/namenode/NamenodeFsck.java | 2 +- .../org/apache/hadoop/hdfs/TestSafeMode.java | 75 +++++++++++++++++++ .../hdfs/server/namenode/NameNodeAdapter.java | 2 +- 5 files changed, 93 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 26e35325d4..d13a247dd9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -316,6 +316,9 @@ Branch-2 ( Unreleased changes ) HDFS-3517. TestStartup should bind ephemeral ports. (eli) + HDFS-3522. If a namenode is in safemode, it should throw SafeModeException + when getBlockLocations has zero locations. (Brandon Li via szetszwo) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 22013e0feb..f965ed04d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1089,7 +1089,8 @@ void setOwner(String src, String username, String group) LocatedBlocks getBlockLocations(String clientMachine, String src, long offset, long length) throws AccessControlException, FileNotFoundException, UnresolvedLinkException, IOException { - LocatedBlocks blocks = getBlockLocations(src, offset, length, true, true); + LocatedBlocks blocks = getBlockLocations(src, offset, length, true, true, + true); if (blocks != null) { blockManager.getDatanodeManager().sortLocatedBlocks( clientMachine, blocks.getLocatedBlocks()); @@ -1103,8 +1104,8 @@ LocatedBlocks getBlockLocations(String clientMachine, String src, * @throws FileNotFoundException, UnresolvedLinkException, IOException */ LocatedBlocks getBlockLocations(String src, long offset, long length, - boolean doAccessTime, boolean needBlockToken) throws FileNotFoundException, - UnresolvedLinkException, IOException { + boolean doAccessTime, boolean needBlockToken, boolean checkSafeMode) + throws FileNotFoundException, UnresolvedLinkException, IOException { if (isPermissionEnabled) { checkPathAccess(src, FsAction.READ); } @@ -1124,6 +1125,15 @@ LocatedBlocks getBlockLocations(String src, long offset, long length, Server.getRemoteIp(), "open", src, null, null); } + if (checkSafeMode && isInSafeMode()) { + for (LocatedBlock b : ret.getLocatedBlocks()) { + // if safemode & no block locations yet then throw safemodeException + if ((b.getLocations() == null) || (b.getLocations().length == 0)) { + throw new SafeModeException("Zero blocklocations for " + src, + safeMode); + } + } + } return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index 2100b2c0cf..dd377cab4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -277,7 +277,7 @@ void check(String parent, HdfsFileStatus file, Result res) throws IOException { // Get block locations without updating the file access time // and without block access tokens LocatedBlocks blocks = namenode.getNamesystem().getBlockLocations(path, 0, - fileLen, false, false); + fileLen, false, false, false); if (blocks == null) { // the file is deleted return; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java index 3da68643d7..d88001240e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.permission.FsPermission; @@ -31,9 +32,11 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.hdfs.server.namenode.SafeModeException; import org.apache.hadoop.test.GenericTestUtils; import static org.junit.Assert.*; @@ -372,4 +375,76 @@ public void testSafeModeUtils() throws IOException { dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); assertFalse("State was expected to be out of safemode.", dfs.isInSafeMode()); } + + @Test + public void testSafeModeWhenZeroBlockLocations() throws IOException { + + try { + Path file1 = new Path("/tmp/testManualSafeMode/file1"); + Path file2 = new Path("/tmp/testManualSafeMode/file2"); + + System.out.println("Created file1 and file2."); + + // create two files with one block each. + DFSTestUtil.createFile(fs, file1, 1000, (short)1, 0); + DFSTestUtil.createFile(fs, file2, 2000, (short)1, 0); + checkGetBlockLocationsWorks(fs, file1); + + NameNode namenode = cluster.getNameNode(); + + // manually set safemode. + dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + assertTrue("should still be in SafeMode", namenode.isInSafeMode()); + // getBlock locations should still work since block locations exists + checkGetBlockLocationsWorks(fs, file1); + dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + assertFalse("should not be in SafeMode", namenode.isInSafeMode()); + + + // Now 2nd part of the tests where there aren't block locations + cluster.shutdownDataNodes(); + cluster.shutdownNameNode(0); + + // now bring up just the NameNode. + cluster.restartNameNode(); + cluster.waitActive(); + + System.out.println("Restarted cluster with just the NameNode"); + + namenode = cluster.getNameNode(); + + assertTrue("No datanode is started. Should be in SafeMode", + namenode.isInSafeMode()); + FileStatus stat = fs.getFileStatus(file1); + try { + fs.getFileBlockLocations(stat, 0, 1000); + assertTrue("Should have got safemode exception", false); + } catch (SafeModeException e) { + // as expected + } catch (RemoteException re) { + if (!re.getClassName().equals(SafeModeException.class.getName())) + assertTrue("Should have got safemode exception", false); + } + + + dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + assertFalse("Should not be in safemode", namenode.isInSafeMode()); + checkGetBlockLocationsWorks(fs, file1); + + } finally { + if(fs != null) fs.close(); + if(cluster!= null) cluster.shutdown(); + } + } + + void checkGetBlockLocationsWorks(FileSystem fs, Path fileName) throws IOException { + FileStatus stat = fs.getFileStatus(fileName); + try { + fs.getFileBlockLocations(stat, 0, 1000); + } catch (SafeModeException e) { + assertTrue("Should have not got safemode exception", false); + } catch (RemoteException re) { + assertTrue("Should have not got safemode exception", false); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java index ab2640d872..6794591e71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java @@ -56,7 +56,7 @@ public static FSNamesystem getNamesystem(NameNode namenode) { public static LocatedBlocks getBlockLocations(NameNode namenode, String src, long offset, long length) throws IOException { return namenode.getNamesystem().getBlockLocations( - src, offset, length, false, true); + src, offset, length, false, true, true); } public static HdfsFileStatus getFileInfo(NameNode namenode, String src, From 7c55d57be4ac369831e5fa2940ea026be91e61d7 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Tue, 12 Jun 2012 04:15:18 +0000 Subject: [PATCH 28/91] HDFS-3049. During the normal NN startup process, fall back on a different edit log if we see one that is corrupt. Contributed by Colin Patrick McCabe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349114 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/server/namenode/FSEditLog.java | 12 - .../hdfs/server/namenode/FSEditLogLoader.java | 4 +- .../hdfs/server/namenode/JournalSet.java | 15 +- .../namenode/RedundantEditLogInputStream.java | 276 ++++++++++++++++++ .../apache/hadoop/hdfs/MiniDFSCluster.java | 70 +++-- .../hdfs/server/namenode/TestEditLog.java | 139 ++++++++- .../namenode/TestEditLogFileOutputStream.java | 10 + .../server/namenode/TestFSEditLogLoader.java | 54 +++- .../namenode/TestFileJournalManager.java | 31 +- .../server/namenode/TestNameNodeRecovery.java | 9 +- 11 files changed, 549 insertions(+), 74 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d13a247dd9..7b27c52afc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -90,6 +90,9 @@ Trunk (unreleased changes) HDFS-3040. TestMulitipleNNDataBlockScanner is misspelled. (Madhukara Phatak via atm) + HDFS-3049. During the normal NN startup process, fall back on a different + edit log if we see one that is corrupt (Colin Patrick McCabe via todd) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index ff38c9594d..c0e8ff0679 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -1173,18 +1173,6 @@ public synchronized Collection selectInputStreams( throw e; } } - // This code will go away as soon as RedundantEditLogInputStream is - // introduced. (HDFS-3049) - try { - if (!streams.isEmpty()) { - streams.get(0).skipUntil(fromTxId); - } - } catch (IOException e) { - // We don't want to throw an exception from here, because that would make - // recovery impossible even if the user requested it. An exception will - // be thrown later, when we don't read the starting txid we expect. - LOG.error("error skipping until transaction " + fromTxId, e); - } return streams; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index 343472dfa1..e1b1eccaa2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -668,7 +668,9 @@ static EditLogValidation validateEditLog(EditLogInputStream in) { FSImage.LOG.warn("Caught exception after reading " + numValid + " ops from " + in + " while determining its valid length." + "Position was " + lastPos, t); - break; + in.resync(); + FSImage.LOG.warn("After resync, position is " + in.getPosition()); + continue; } if (lastTxId == HdfsConstants.INVALID_TXID || op.getTransactionId() > lastTxId) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java index 3d2e23bf2d..92c3e52381 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java @@ -24,7 +24,9 @@ import java.util.Comparator; import java.util.LinkedList; import java.util.List; +import java.util.PriorityQueue; import java.util.SortedSet; +import java.util.TreeSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -40,7 +42,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimaps; import com.google.common.collect.Sets; -import com.google.common.collect.TreeMultiset; /** * Manages a collection of Journals. None of the methods are synchronized, it is @@ -222,8 +223,9 @@ public void apply(JournalAndStream jas) throws IOException { @Override public void selectInputStreams(Collection streams, long fromTxId, boolean inProgressOk) { - final TreeMultiset allStreams = - TreeMultiset.create(EDIT_LOG_INPUT_STREAM_COMPARATOR); + final PriorityQueue allStreams = + new PriorityQueue(64, + EDIT_LOG_INPUT_STREAM_COMPARATOR); for (JournalAndStream jas : journals) { if (jas.isDisabled()) { LOG.info("Skipping jas " + jas + " since it's disabled"); @@ -239,7 +241,8 @@ public void selectInputStreams(Collection streams, // transaction ID. LinkedList acc = new LinkedList(); - for (EditLogInputStream elis : allStreams) { + EditLogInputStream elis; + while ((elis = allStreams.poll()) != null) { if (acc.isEmpty()) { acc.add(elis); } else { @@ -247,7 +250,7 @@ public void selectInputStreams(Collection streams, if (accFirstTxId == elis.getFirstTxId()) { acc.add(elis); } else if (accFirstTxId < elis.getFirstTxId()) { - streams.add(acc.get(0)); + streams.add(new RedundantEditLogInputStream(acc, fromTxId)); acc.clear(); acc.add(elis); } else if (accFirstTxId > elis.getFirstTxId()) { @@ -258,7 +261,7 @@ public void selectInputStreams(Collection streams, } } if (!acc.isEmpty()) { - streams.add(acc.get(0)); + streams.add(new RedundantEditLogInputStream(acc, fromTxId)); acc.clear(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java new file mode 100644 index 0000000000..7a30869290 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java @@ -0,0 +1,276 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.io.IOUtils; + +import com.google.common.base.Preconditions; +import com.google.common.primitives.Longs; + +/** + * A merged input stream that handles failover between different edit logs. + * + * We will currently try each edit log stream exactly once. In other words, we + * don't handle the "ping pong" scenario where different edit logs contain a + * different subset of the available edits. + */ +class RedundantEditLogInputStream extends EditLogInputStream { + public static final Log LOG = LogFactory.getLog(EditLogInputStream.class.getName()); + private int curIdx; + private long prevTxId; + private final EditLogInputStream[] streams; + + /** + * States that the RedundantEditLogInputStream can be in. + * + * 

+   *                   start (if no streams)
+   *                           |
+   *                           V
+   * PrematureEOFException  +----------------+
+   *        +-------------->| EOF            |<--------------+
+   *        |               +----------------+               |
+   *        |                                                |
+   *        |          start (if there are streams)          |
+   *        |                  |                             |
+   *        |                  V                             | EOF
+   *        |   resync      +----------------+ skipUntil  +---------+
+   *        |   +---------->| SKIP_UNTIL     |----------->|  OK     |
+   *        |   |           +----------------+            +---------+
+   *        |   |                | IOE   ^ fail over to      | IOE
+   *        |   |                V       | next stream       |
+   * +----------------------+   +----------------+           |
+   * | STREAM_FAILED_RESYNC |   | STREAM_FAILED  |<----------+
+   * +----------------------+   +----------------+
+   *                  ^   Recovery mode    |
+   *                  +--------------------+
+   *
+ */ + static private enum State { + /** We need to skip until prevTxId + 1 */ + SKIP_UNTIL, + /** We're ready to read opcodes out of the current stream */ + OK, + /** The current stream has failed. */ + STREAM_FAILED, + /** The current stream has failed, and resync() was called. */ + STREAM_FAILED_RESYNC, + /** There are no more opcodes to read from this + * RedundantEditLogInputStream */ + EOF; + } + + private State state; + private IOException prevException; + + RedundantEditLogInputStream(Collection streams, + long startTxId) { + this.curIdx = 0; + this.prevTxId = (startTxId == HdfsConstants.INVALID_TXID) ? + HdfsConstants.INVALID_TXID : (startTxId - 1); + this.state = (streams.isEmpty()) ? State.EOF : State.SKIP_UNTIL; + this.prevException = null; + // EditLogInputStreams in a RedundantEditLogInputStream must be finalized, + // and can't be pre-transactional. + EditLogInputStream first = null; + for (EditLogInputStream s : streams) { + Preconditions.checkArgument(s.getFirstTxId() != + HdfsConstants.INVALID_TXID, "invalid first txid in stream: %s", s); + Preconditions.checkArgument(s.getLastTxId() != + HdfsConstants.INVALID_TXID, "invalid last txid in stream: %s", s); + if (first == null) { + first = s; + } else { + Preconditions.checkArgument(s.getFirstTxId() == first.getFirstTxId(), + "All streams in the RedundantEditLogInputStream must have the same " + + "start transaction ID! " + first + " had start txId " + + first.getFirstTxId() + ", but " + s + " had start txId " + + s.getFirstTxId()); + } + } + + this.streams = streams.toArray(new EditLogInputStream[0]); + + // We sort the streams here so that the streams that end later come first. + Arrays.sort(this.streams, new Comparator() { + @Override + public int compare(EditLogInputStream a, EditLogInputStream b) { + return Longs.compare(b.getLastTxId(), a.getLastTxId()); + } + }); + } + + @Override + public String getName() { + StringBuilder bld = new StringBuilder(); + String prefix = ""; + for (EditLogInputStream elis : streams) { + bld.append(prefix); + bld.append(elis.getName()); + prefix = ", "; + } + return bld.toString(); + } + + @Override + public long getFirstTxId() { + return streams[curIdx].getFirstTxId(); + } + + @Override + public long getLastTxId() { + return streams[curIdx].getLastTxId(); + } + + @Override + public void close() throws IOException { + IOUtils.cleanup(LOG, streams); + } + + @Override + protected FSEditLogOp nextValidOp() { + try { + if (state == State.STREAM_FAILED) { + state = State.STREAM_FAILED_RESYNC; + } + return nextOp(); + } catch (IOException e) { + return null; + } + } + + @Override + protected FSEditLogOp nextOp() throws IOException { + while (true) { + switch (state) { + case SKIP_UNTIL: + try { + if (prevTxId != HdfsConstants.INVALID_TXID) { + LOG.info("Fast-forwarding stream '" + streams[curIdx].getName() + + "' to transaction ID " + (prevTxId + 1)); + streams[curIdx].skipUntil(prevTxId + 1); + } + } catch (IOException e) { + prevException = e; + state = State.STREAM_FAILED; + } + state = State.OK; + break; + case OK: + try { + FSEditLogOp op = streams[curIdx].readOp(); + if (op == null) { + state = State.EOF; + if (streams[curIdx].getLastTxId() == prevTxId) { + return null; + } else { + throw new PrematureEOFException("got premature end-of-file " + + "at txid " + prevTxId + "; expected file to go up to " + + streams[curIdx].getLastTxId()); + } + } + prevTxId = op.getTransactionId(); + return op; + } catch (IOException e) { + prevException = e; + state = State.STREAM_FAILED; + } + break; + case STREAM_FAILED: + if (curIdx + 1 == streams.length) { + throw prevException; + } + long oldLast = streams[curIdx].getLastTxId(); + long newLast = streams[curIdx + 1].getLastTxId(); + if (newLast < oldLast) { + throw new IOException("We encountered an error reading " + + streams[curIdx].getName() + ". During automatic edit log " + + "failover, we noticed that all of the remaining edit log " + + "streams are shorter than the current one! The best " + + "remaining edit log ends at transaction " + + newLast + ", but we thought we could read up to transaction " + + oldLast + ". If you continue, metadata will be lost forever!"); + } + LOG.error("Got error reading edit log input stream " + + streams[curIdx].getName() + "; failing over to edit log " + + streams[curIdx + 1].getName(), prevException); + curIdx++; + state = State.SKIP_UNTIL; + break; + case STREAM_FAILED_RESYNC: + if (curIdx + 1 == streams.length) { + if (prevException instanceof PrematureEOFException) { + // bypass early EOF check + state = State.EOF; + } else { + streams[curIdx].resync(); + state = State.SKIP_UNTIL; + } + } else { + LOG.error("failing over to edit log " + + streams[curIdx + 1].getName()); + curIdx++; + state = State.SKIP_UNTIL; + } + break; + case EOF: + return null; + } + } + } + + @Override + public int getVersion() throws IOException { + return streams[curIdx].getVersion(); + } + + @Override + public long getPosition() { + return streams[curIdx].getPosition(); + } + + @Override + public long length() throws IOException { + return streams[curIdx].length(); + } + + @Override + public boolean isInProgress() { + return streams[curIdx].isInProgress(); + } + + static private final class PrematureEOFException extends IOException { + private static final long serialVersionUID = 1L; + PrematureEOFException(String msg) { + super(msg); + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index d57f792472..5c9c465997 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -134,6 +134,7 @@ public static class Builder { private boolean format = true; private boolean manageNameDfsDirs = true; private boolean manageNameDfsSharedDirs = true; + private boolean enableManagedDfsDirsRedundancy = true; private boolean manageDataDfsDirs = true; private StartupOption option = null; private String[] racks = null; @@ -187,7 +188,7 @@ public Builder manageNameDfsDirs(boolean val) { this.manageNameDfsDirs = val; return this; } - + /** * Default: true */ @@ -196,6 +197,14 @@ public Builder manageNameDfsSharedDirs(boolean val) { return this; } + /** + * Default: true + */ + public Builder enableManagedDfsDirsRedundancy(boolean val) { + this.enableManagedDfsDirsRedundancy = val; + return this; + } + /** * Default: true */ @@ -298,6 +307,7 @@ private MiniDFSCluster(Builder builder) throws IOException { builder.format, builder.manageNameDfsDirs, builder.manageNameDfsSharedDirs, + builder.enableManagedDfsDirsRedundancy, builder.manageDataDfsDirs, builder.option, builder.racks, @@ -385,7 +395,7 @@ public MiniDFSCluster() { public MiniDFSCluster(Configuration conf, int numDataNodes, StartupOption nameNodeOperation) throws IOException { - this(0, conf, numDataNodes, false, false, false, nameNodeOperation, + this(0, conf, numDataNodes, false, false, false, false, nameNodeOperation, null, null, null); } @@ -407,7 +417,8 @@ public MiniDFSCluster(Configuration conf, int numDataNodes, boolean format, String[] racks) throws IOException { - this(0, conf, numDataNodes, format, true, true, null, racks, null, null); + this(0, conf, numDataNodes, format, true, true, true, null, + racks, null, null); } /** @@ -429,7 +440,8 @@ public MiniDFSCluster(Configuration conf, int numDataNodes, boolean format, String[] racks, String[] hosts) throws IOException { - this(0, conf, numDataNodes, format, true, true, null, racks, hosts, null); + this(0, conf, numDataNodes, format, true, true, true, null, + racks, hosts, null); } /** @@ -462,8 +474,8 @@ public MiniDFSCluster(int nameNodePort, boolean manageDfsDirs, StartupOption operation, String[] racks) throws IOException { - this(nameNodePort, conf, numDataNodes, format, manageDfsDirs, manageDfsDirs, - operation, racks, null, null); + this(nameNodePort, conf, numDataNodes, format, manageDfsDirs, + manageDfsDirs, manageDfsDirs, operation, racks, null, null); } /** @@ -497,7 +509,7 @@ public MiniDFSCluster(int nameNodePort, String[] racks, long[] simulatedCapacities) throws IOException { this(nameNodePort, conf, numDataNodes, format, manageDfsDirs, manageDfsDirs, - operation, racks, null, simulatedCapacities); + manageDfsDirs, operation, racks, null, simulatedCapacities); } /** @@ -531,13 +543,15 @@ public MiniDFSCluster(int nameNodePort, int numDataNodes, boolean format, boolean manageNameDfsDirs, + boolean enableManagedDfsDirsRedundancy, boolean manageDataDfsDirs, StartupOption operation, String[] racks, String hosts[], long[] simulatedCapacities) throws IOException { this.nameNodes = new NameNodeInfo[1]; // Single namenode in the cluster initMiniDFSCluster(conf, numDataNodes, format, - manageNameDfsDirs, true, manageDataDfsDirs, operation, racks, hosts, + manageNameDfsDirs, true, enableManagedDfsDirsRedundancy, manageDataDfsDirs, + operation, racks, hosts, simulatedCapacities, null, true, false, MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0)); } @@ -545,8 +559,8 @@ public MiniDFSCluster(int nameNodePort, private void initMiniDFSCluster( Configuration conf, int numDataNodes, boolean format, boolean manageNameDfsDirs, - boolean manageNameDfsSharedDirs, boolean manageDataDfsDirs, - StartupOption operation, String[] racks, + boolean manageNameDfsSharedDirs, boolean enableManagedDfsDirsRedundancy, + boolean manageDataDfsDirs, StartupOption operation, String[] racks, String[] hosts, long[] simulatedCapacities, String clusterId, boolean waitSafeMode, boolean setupHostsFile, MiniDFSNNTopology nnTopology) @@ -586,6 +600,7 @@ private void initMiniDFSCluster( federation = nnTopology.isFederated(); createNameNodesAndSetConf( nnTopology, manageNameDfsDirs, manageNameDfsSharedDirs, + enableManagedDfsDirsRedundancy, format, operation, clusterId, conf); if (format) { @@ -608,7 +623,8 @@ private void initMiniDFSCluster( private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology, boolean manageNameDfsDirs, boolean manageNameDfsSharedDirs, - boolean format, StartupOption operation, String clusterId, + boolean enableManagedDfsDirsRedundancy, boolean format, + StartupOption operation, String clusterId, Configuration conf) throws IOException { Preconditions.checkArgument(nnTopology.countNameNodes() > 0, "empty NN topology: no namenodes specified!"); @@ -664,7 +680,7 @@ private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology, Collection prevNNDirs = null; int nnCounterForFormat = nnCounter; for (NNConf nn : nameservice.getNNs()) { - initNameNodeConf(conf, nsId, nn.getNnId(), manageNameDfsDirs, + initNameNodeConf(conf, nsId, nn.getNnId(), manageNameDfsDirs, manageNameDfsDirs, nnCounterForFormat); Collection namespaceDirs = FSNamesystem.getNamespaceDirs(conf); if (format) { @@ -696,7 +712,8 @@ private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology, // Start all Namenodes for (NNConf nn : nameservice.getNNs()) { - initNameNodeConf(conf, nsId, nn.getNnId(), manageNameDfsDirs, nnCounter); + initNameNodeConf(conf, nsId, nn.getNnId(), manageNameDfsDirs, + enableManagedDfsDirsRedundancy, nnCounter); createNameNode(nnCounter++, conf, numDataNodes, false, operation, clusterId, nsId, nn.getNnId()); } @@ -721,8 +738,8 @@ public NameNodeInfo[] getNameNodeInfos() { private void initNameNodeConf(Configuration conf, String nameserviceId, String nnId, - boolean manageNameDfsDirs, int nnIndex) - throws IOException { + boolean manageNameDfsDirs, boolean enableManagedDfsDirsRedundancy, + int nnIndex) throws IOException { if (nameserviceId != null) { conf.set(DFS_NAMESERVICE_ID, nameserviceId); } @@ -731,12 +748,21 @@ private void initNameNodeConf(Configuration conf, } if (manageNameDfsDirs) { - conf.set(DFS_NAMENODE_NAME_DIR_KEY, - fileAsURI(new File(base_dir, "name" + (2*nnIndex + 1)))+","+ - fileAsURI(new File(base_dir, "name" + (2*nnIndex + 2)))); - conf.set(DFS_NAMENODE_CHECKPOINT_DIR_KEY, - fileAsURI(new File(base_dir, "namesecondary" + (2*nnIndex + 1)))+","+ - fileAsURI(new File(base_dir, "namesecondary" + (2*nnIndex + 2)))); + if (enableManagedDfsDirsRedundancy) { + conf.set(DFS_NAMENODE_NAME_DIR_KEY, + fileAsURI(new File(base_dir, "name" + (2*nnIndex + 1)))+","+ + fileAsURI(new File(base_dir, "name" + (2*nnIndex + 2)))); + conf.set(DFS_NAMENODE_CHECKPOINT_DIR_KEY, + fileAsURI(new File(base_dir, "namesecondary" + (2*nnIndex + 1)))+","+ + fileAsURI(new File(base_dir, "namesecondary" + (2*nnIndex + 2)))); + } else { + conf.set(DFS_NAMENODE_NAME_DIR_KEY, + fileAsURI(new File(base_dir, "name" + (2*nnIndex + 1))). + toString()); + conf.set(DFS_NAMENODE_CHECKPOINT_DIR_KEY, + fileAsURI(new File(base_dir, "namesecondary" + (2*nnIndex + 1))). + toString()); + } } } @@ -2134,7 +2160,7 @@ public NameNode addNameNode(Configuration conf, int namenodePort) String nnId = null; initNameNodeAddress(conf, nameserviceId, new NNConf(nnId).setIpcPort(namenodePort)); - initNameNodeConf(conf, nameserviceId, nnId, true, nnIndex); + initNameNodeConf(conf, nameserviceId, nnId, true, true, nnIndex); createNameNode(nnIndex, conf, numDataNodes, true, null, null, nameserviceId, nnId); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java index 5e77d730b6..ee037cb2d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java @@ -506,21 +506,29 @@ public void testEditChecksum() throws Exception { FSImage fsimage = namesystem.getFSImage(); final FSEditLog editLog = fsimage.getEditLog(); fileSys.mkdirs(new Path("/tmp")); - StorageDirectory sd = fsimage.getStorage().dirIterator(NameNodeDirType.EDITS).next(); + + Iterator iter = fsimage.getStorage(). + dirIterator(NameNodeDirType.EDITS); + LinkedList sds = new LinkedList(); + while (iter.hasNext()) { + sds.add(iter.next()); + } editLog.close(); cluster.shutdown(); - File editFile = NNStorage.getFinalizedEditsFile(sd, 1, 3); - assertTrue(editFile.exists()); - - long fileLen = editFile.length(); - System.out.println("File name: " + editFile + " len: " + fileLen); - RandomAccessFile rwf = new RandomAccessFile(editFile, "rw"); - rwf.seek(fileLen-4); // seek to checksum bytes - int b = rwf.readInt(); - rwf.seek(fileLen-4); - rwf.writeInt(b+1); - rwf.close(); + for (StorageDirectory sd : sds) { + File editFile = NNStorage.getFinalizedEditsFile(sd, 1, 3); + assertTrue(editFile.exists()); + + long fileLen = editFile.length(); + LOG.debug("Corrupting Log File: " + editFile + " len: " + fileLen); + RandomAccessFile rwf = new RandomAccessFile(editFile, "rw"); + rwf.seek(fileLen-4); // seek to checksum bytes + int b = rwf.readInt(); + rwf.seek(fileLen-4); + rwf.writeInt(b+1); + rwf.close(); + } try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).format(false).build(); @@ -1232,6 +1240,113 @@ public void testFuzzSequences() throws IOException { } } + private static long readAllEdits(Collection streams, + long startTxId) throws IOException { + FSEditLogOp op; + long nextTxId = startTxId; + long numTx = 0; + for (EditLogInputStream s : streams) { + while (true) { + op = s.readOp(); + if (op == null) + break; + if (op.getTransactionId() != nextTxId) { + throw new IOException("out of order transaction ID! expected " + + nextTxId + " but got " + op.getTransactionId() + " when " + + "reading " + s.getName()); + } + numTx++; + nextTxId = op.getTransactionId() + 1; + } + } + return numTx; + } + + /** + * Test edit log failover. If a single edit log is missing, other + * edits logs should be used instead. + */ + @Test + public void testEditLogFailOverFromMissing() throws IOException { + File f1 = new File(TEST_DIR + "/failover0"); + File f2 = new File(TEST_DIR + "/failover1"); + List editUris = ImmutableList.of(f1.toURI(), f2.toURI()); + + NNStorage storage = setupEdits(editUris, 3); + + final long startErrorTxId = 1*TXNS_PER_ROLL + 1; + final long endErrorTxId = 2*TXNS_PER_ROLL; + + File[] files = new File(f1, "current").listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + if (name.startsWith(NNStorage.getFinalizedEditsFileName(startErrorTxId, + endErrorTxId))) { + return true; + } + return false; + } + }); + assertEquals(1, files.length); + assertTrue(files[0].delete()); + + FSEditLog editlog = getFSEditLog(storage); + editlog.initJournalsForWrite(); + long startTxId = 1; + try { + readAllEdits(editlog.selectInputStreams(startTxId, 4*TXNS_PER_ROLL), + startTxId); + } catch (IOException e) { + LOG.error("edit log failover didn't work", e); + fail("Edit log failover didn't work"); + } + } + + /** + * Test edit log failover from a corrupt edit log + */ + @Test + public void testEditLogFailOverFromCorrupt() throws IOException { + File f1 = new File(TEST_DIR + "/failover0"); + File f2 = new File(TEST_DIR + "/failover1"); + List editUris = ImmutableList.of(f1.toURI(), f2.toURI()); + + NNStorage storage = setupEdits(editUris, 3); + + final long startErrorTxId = 1*TXNS_PER_ROLL + 1; + final long endErrorTxId = 2*TXNS_PER_ROLL; + + File[] files = new File(f1, "current").listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + if (name.startsWith(NNStorage.getFinalizedEditsFileName(startErrorTxId, + endErrorTxId))) { + return true; + } + return false; + } + }); + assertEquals(1, files.length); + + long fileLen = files[0].length(); + LOG.debug("Corrupting Log File: " + files[0] + " len: " + fileLen); + RandomAccessFile rwf = new RandomAccessFile(files[0], "rw"); + rwf.seek(fileLen-4); // seek to checksum bytes + int b = rwf.readInt(); + rwf.seek(fileLen-4); + rwf.writeInt(b+1); + rwf.close(); + + FSEditLog editlog = getFSEditLog(storage); + editlog.initJournalsForWrite(); + long startTxId = 1; + try { + readAllEdits(editlog.selectInputStreams(startTxId, 4*TXNS_PER_ROLL), + startTxId); + } catch (IOException e) { + LOG.error("edit log failover didn't work", e); + fail("Edit log failover didn't work"); + } + } + /** * Test creating a directory with lots and lots of edit log segments */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java index d39df4030d..c1ca4ac09c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java @@ -50,6 +50,16 @@ public void deleteEditsFile() { TEST_EDITS.delete(); } + @Test + public void testConstants() { + // Each call to FSEditLogOp#Reader#readOp can read at most MAX_OP_SIZE bytes + // before getting an exception. So we don't want to preallocate a longer + // region than MAX_OP_SIZE, because then we'd get an IOException when reading + // through the padding at the end of the file. + assertTrue(EditLogFileOutputStream.PREALLOCATION_LENGTH < + FSEditLogOp.MAX_OP_SIZE); + } + @Test public void testPreallocation() throws IOException { Configuration conf = new HdfsConfiguration(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java index a54df2ca81..267e128413 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java @@ -77,7 +77,7 @@ public void testDisplayRecentEditLogOpCodes() throws IOException { MiniDFSCluster cluster = null; FileSystem fileSys = null; cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES) - .build(); + .enableManagedDfsDirsRedundancy(false).build(); cluster.waitActive(); fileSys = cluster.getFileSystem(); final FSNamesystem namesystem = cluster.getNamesystem(); @@ -107,7 +107,7 @@ public void testDisplayRecentEditLogOpCodes() throws IOException { bld.append("Recent opcode offsets: (\\d+\\s*){4}$"); try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES) - .format(false).build(); + .enableManagedDfsDirsRedundancy(false).format(false).build(); fail("should not be able to start"); } catch (IOException e) { assertTrue("error message contains opcodes message", @@ -326,6 +326,56 @@ public void testValidateEditLogWithCorruptHeader() throws IOException { assertTrue(validation.hasCorruptHeader()); } + @Test + public void testValidateEditLogWithCorruptBody() throws IOException { + File testDir = new File(TEST_DIR, "testValidateEditLogWithCorruptBody"); + SortedMap offsetToTxId = Maps.newTreeMap(); + final int NUM_TXNS = 20; + File logFile = prepareUnfinalizedTestEditLog(testDir, NUM_TXNS, + offsetToTxId); + // Back up the uncorrupted log + File logFileBak = new File(testDir, logFile.getName() + ".bak"); + Files.copy(logFile, logFileBak); + EditLogValidation validation = + EditLogFileInputStream.validateEditLog(logFile); + assertTrue(!validation.hasCorruptHeader()); + // We expect that there will be an OP_START_LOG_SEGMENT, followed by + // NUM_TXNS opcodes, followed by an OP_END_LOG_SEGMENT. + assertEquals(NUM_TXNS + 1, validation.getEndTxId()); + // Corrupt each edit and verify that validation continues to work + for (Map.Entry entry : offsetToTxId.entrySet()) { + long txOffset = entry.getKey(); + long txId = entry.getValue(); + + // Restore backup, corrupt the txn opcode + Files.copy(logFileBak, logFile); + corruptByteInFile(logFile, txOffset); + validation = EditLogFileInputStream.validateEditLog(logFile); + long expectedEndTxId = (txId == (NUM_TXNS + 1)) ? + NUM_TXNS : (NUM_TXNS + 1); + assertEquals("Failed when corrupting txn opcode at " + txOffset, + expectedEndTxId, validation.getEndTxId()); + assertTrue(!validation.hasCorruptHeader()); + } + + // Truncate right before each edit and verify that validation continues + // to work + for (Map.Entry entry : offsetToTxId.entrySet()) { + long txOffset = entry.getKey(); + long txId = entry.getValue(); + + // Restore backup, corrupt the txn opcode + Files.copy(logFileBak, logFile); + truncateFile(logFile, txOffset); + validation = EditLogFileInputStream.validateEditLog(logFile); + long expectedEndTxId = (txId == 0) ? + HdfsConstants.INVALID_TXID : (txId - 1); + assertEquals("Failed when corrupting txid " + txId + " txn opcode " + + "at " + txOffset, expectedEndTxId, validation.getEndTxId()); + assertTrue(!validation.hasCorruptHeader()); + } + } + @Test public void testValidateEmptyEditLog() throws IOException { File testDir = new File(TEST_DIR, "testValidateEmptyEditLog"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java index e972f599b9..80d79585af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java @@ -20,10 +20,10 @@ import static org.junit.Assert.*; import java.net.URI; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Iterator; +import java.util.PriorityQueue; import java.io.RandomAccessFile; import java.io.File; @@ -33,7 +33,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.JournalManager.CorruptionException; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; @@ -45,7 +44,6 @@ import static org.apache.hadoop.hdfs.server.namenode.TestEditLog.TXNS_PER_FAIL; import com.google.common.collect.ImmutableList; -import com.google.common.collect.TreeMultiset; import com.google.common.base.Joiner; public class TestFileJournalManager { @@ -64,12 +62,13 @@ public class TestFileJournalManager { static long getNumberOfTransactions(FileJournalManager jm, long fromTxId, boolean inProgressOk, boolean abortOnGap) throws IOException { long numTransactions = 0, txId = fromTxId; - final TreeMultiset allStreams = - TreeMultiset.create(JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); + final PriorityQueue allStreams = + new PriorityQueue(64, + JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); jm.selectInputStreams(allStreams, fromTxId, inProgressOk); - + EditLogInputStream elis = null; try { - for (EditLogInputStream elis : allStreams) { + while ((elis = allStreams.poll()) != null) { elis.skipUntil(txId); while (true) { FSEditLogOp op = elis.readOp(); @@ -87,6 +86,7 @@ static long getNumberOfTransactions(FileJournalManager jm, long fromTxId, } } finally { IOUtils.cleanup(LOG, allStreams.toArray(new EditLogInputStream[0])); + IOUtils.cleanup(LOG, elis); } return numTransactions; } @@ -379,27 +379,28 @@ public void testMatchEditLogInvalidDirThrowsIOException() throws IOException { private static EditLogInputStream getJournalInputStream(JournalManager jm, long txId, boolean inProgressOk) throws IOException { - final TreeMultiset allStreams = - TreeMultiset.create(JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); + final PriorityQueue allStreams = + new PriorityQueue(64, + JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); jm.selectInputStreams(allStreams, txId, inProgressOk); + EditLogInputStream elis = null, ret; try { - for (Iterator iter = allStreams.iterator(); - iter.hasNext();) { - EditLogInputStream elis = iter.next(); + while ((elis = allStreams.poll()) != null) { if (elis.getFirstTxId() > txId) { break; } if (elis.getLastTxId() < txId) { - iter.remove(); elis.close(); continue; } elis.skipUntil(txId); - iter.remove(); - return elis; + ret = elis; + elis = null; + return ret; } } finally { IOUtils.cleanup(LOG, allStreams.toArray(new EditLogInputStream[0])); + IOUtils.cleanup(LOG, elis); } return null; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java index 608ee26e3f..c1287e7d5d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java @@ -343,7 +343,7 @@ static void testNameNodeRecoveryImpl(Corruptor corruptor, boolean finalize) StorageDirectory sd = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) - .build(); + .enableManagedDfsDirsRedundancy(false).build(); cluster.waitActive(); if (!finalize) { // Normally, the in-progress edit log would be finalized by @@ -379,7 +379,7 @@ static void testNameNodeRecoveryImpl(Corruptor corruptor, boolean finalize) try { LOG.debug("trying to start normally (this should fail)..."); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) - .format(false).build(); + .enableManagedDfsDirsRedundancy(false).format(false).build(); cluster.waitActive(); cluster.shutdown(); if (needRecovery) { @@ -404,7 +404,8 @@ static void testNameNodeRecoveryImpl(Corruptor corruptor, boolean finalize) try { LOG.debug("running recovery..."); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) - .format(false).startupOption(recoverStartOpt).build(); + .enableManagedDfsDirsRedundancy(false).format(false) + .startupOption(recoverStartOpt).build(); } catch (IOException e) { fail("caught IOException while trying to recover. " + "message was " + e.getMessage() + @@ -420,7 +421,7 @@ static void testNameNodeRecoveryImpl(Corruptor corruptor, boolean finalize) try { LOG.debug("starting cluster normally after recovery..."); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) - .format(false).build(); + .enableManagedDfsDirsRedundancy(false).format(false).build(); LOG.debug("successfully recovered the " + corruptor.getName() + " corrupted edit log"); cluster.waitActive(); From 553b8c09cca8bb4f676c6a99e389d2f42c197fa5 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Tue, 12 Jun 2012 05:00:07 +0000 Subject: [PATCH 29/91] HDFS-3520. Add transfer rate logging to TransferFsImage. Contributed by Eli Collins git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349117 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hadoop/hdfs/server/namenode/TransferFsImage.java | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7b27c52afc..5432479fb6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -220,6 +220,8 @@ Branch-2 ( Unreleased changes ) HDFS-3052. Change INodeFile and INodeFileUnderConstruction to package private. (szetszwo) + HDFS-3520. Add transfer rate logging to TransferFsImage. (eli) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java index 97088c5f43..492cf28fbe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java @@ -32,11 +32,11 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.io.MD5Hash; -import org.apache.hadoop.security.UserGroupInformation; import com.google.common.collect.Lists; @@ -207,6 +207,7 @@ static MD5Hash getFileClient(String nnHostPort, // // open connection to remote server // + long startTime = Util.monotonicNow(); URL url = new URL(str); HttpURLConnection connection = (HttpURLConnection) @@ -312,6 +313,11 @@ static MD5Hash getFileClient(String nnHostPort, advertisedSize); } } + double xferSec = Math.max( + ((float)(Util.monotonicNow() - startTime)) / 1000.0, 0.001); + long xferKb = received / 1024; + LOG.info(String.format("Transfer took %.2fs at %.2f KB/s", + xferSec, xferKb / xferSec)); if (digester != null) { MD5Hash computedDigest = new MD5Hash(digester.digest()); From 76dd6b0830b4c0a87767adc6c6d077637579082b Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 12 Jun 2012 05:27:15 +0000 Subject: [PATCH 30/91] HDFS-3504. Support configurable retry policy in DFSClient for RPC connections and RPC calls, and add MultipleLinearRandomRetry, a new retry policy. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349124 13f79535-47bb-0310-9956-ffa450edef68 --- .../io/retry/RetryInvocationHandler.java | 17 +- .../apache/hadoop/io/retry/RetryPolicies.java | 242 +++++++++++++++++- .../apache/hadoop/io/retry/RetryPolicy.java | 6 + .../apache/hadoop/io/retry/RetryProxy.java | 11 +- .../java/org/apache/hadoop/ipc/Client.java | 122 ++++++--- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 28 +- .../main/java/org/apache/hadoop/ipc/RPC.java | 19 +- .../apache/hadoop/ipc/RemoteException.java | 5 +- .../java/org/apache/hadoop/ipc/RpcEngine.java | 4 +- .../apache/hadoop/ipc/WritableRpcEngine.java | 8 +- .../java/org/apache/hadoop/ipc/TestRPC.java | 30 ++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 + .../apache/hadoop/hdfs/NameNodeProxies.java | 113 +++++++- .../apache/hadoop/hdfs/MiniDFSCluster.java | 11 +- .../hadoop/hdfs/TestDFSClientRetries.java | 171 ++++++++++++- 16 files changed, 675 insertions(+), 120 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java index 323542cbd3..381ce3e501 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java @@ -33,7 +33,7 @@ class RetryInvocationHandler implements RpcInvocationHandler { public static final Log LOG = LogFactory.getLog(RetryInvocationHandler.class); - private FailoverProxyProvider proxyProvider; + private final FailoverProxyProvider proxyProvider; /** * The number of times the associated proxyProvider has ever been failed over. @@ -41,26 +41,25 @@ class RetryInvocationHandler implements RpcInvocationHandler { private long proxyProviderFailoverCount = 0; private volatile boolean hasMadeASuccessfulCall = false; - private RetryPolicy defaultPolicy; - private Map methodNameToPolicyMap; + private final RetryPolicy defaultPolicy; + private final Map methodNameToPolicyMap; private Object currentProxy; public RetryInvocationHandler(FailoverProxyProvider proxyProvider, RetryPolicy retryPolicy) { - this.proxyProvider = proxyProvider; - this.defaultPolicy = retryPolicy; - this.methodNameToPolicyMap = Collections.emptyMap(); - this.currentProxy = proxyProvider.getProxy(); + this(proxyProvider, retryPolicy, Collections.emptyMap()); } - + public RetryInvocationHandler(FailoverProxyProvider proxyProvider, + RetryPolicy defaultPolicy, Map methodNameToPolicyMap) { this.proxyProvider = proxyProvider; - this.defaultPolicy = RetryPolicies.TRY_ONCE_THEN_FAIL; + this.defaultPolicy = defaultPolicy; this.methodNameToPolicyMap = methodNameToPolicyMap; this.currentProxy = proxyProvider.getProxy(); } + @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { RetryPolicy policy = methodNameToPolicyMap.get(method.getName()); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java index 2be8b75999..8b8387ce2c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java @@ -22,10 +22,13 @@ import java.net.NoRouteToHostException; import java.net.SocketException; import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.Random; import java.util.Map.Entry; +import java.util.Random; import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; @@ -33,8 +36,6 @@ import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.StandbyException; -import com.google.common.annotations.VisibleForTesting; - /** *

* A collection of useful implementations of {@link RetryPolicy}. @@ -44,7 +45,12 @@ public class RetryPolicies { public static final Log LOG = LogFactory.getLog(RetryPolicies.class); - private static final Random RAND = new Random(); + private static ThreadLocal RANDOM = new ThreadLocal() { + @Override + protected Random initialValue() { + return new Random(); + } + }; /** *

@@ -157,17 +163,35 @@ public RetryAction shouldRetry(Exception e, int retries, int failovers, } } + /** + * Retry up to maxRetries. + * The actual sleep time of the n-th retry is f(n, sleepTime), + * where f is a function provided by the subclass implementation. + * + * The object of the subclasses should be immutable; + * otherwise, the subclass must override hashCode(), equals(..) and toString(). + */ static abstract class RetryLimited implements RetryPolicy { - int maxRetries; - long sleepTime; - TimeUnit timeUnit; + final int maxRetries; + final long sleepTime; + final TimeUnit timeUnit; - public RetryLimited(int maxRetries, long sleepTime, TimeUnit timeUnit) { + private String myString; + + RetryLimited(int maxRetries, long sleepTime, TimeUnit timeUnit) { + if (maxRetries < 0) { + throw new IllegalArgumentException("maxRetries = " + maxRetries+" < 0"); + } + if (sleepTime < 0) { + throw new IllegalArgumentException("sleepTime = " + sleepTime + " < 0"); + } + this.maxRetries = maxRetries; this.sleepTime = sleepTime; this.timeUnit = timeUnit; } + @Override public RetryAction shouldRetry(Exception e, int retries, int failovers, boolean isMethodIdempotent) throws Exception { if (retries >= maxRetries) { @@ -178,6 +202,30 @@ public RetryAction shouldRetry(Exception e, int retries, int failovers, } protected abstract long calculateSleepTime(int retries); + + @Override + public int hashCode() { + return toString().hashCode(); + } + + @Override + public boolean equals(final Object that) { + if (this == that) { + return true; + } else if (that == null || this.getClass() != that.getClass()) { + return false; + } + return this.toString().equals(that.toString()); + } + + @Override + public String toString() { + if (myString == null) { + myString = getClass().getSimpleName() + "(maxRetries=" + maxRetries + + ", sleepTime=" + sleepTime + " " + timeUnit + ")"; + } + return myString; + } } static class RetryUpToMaximumCountWithFixedSleep extends RetryLimited { @@ -208,6 +256,169 @@ protected long calculateSleepTime(int retries) { } } + /** + * Given pairs of number of retries and sleep time (n0, t0), (n1, t1), ..., + * the first n0 retries sleep t0 milliseconds on average, + * the following n1 retries sleep t1 milliseconds on average, and so on. + * + * For all the sleep, the actual sleep time is randomly uniform distributed + * in the close interval [0.5t, 1.5t], where t is the sleep time specified. + * + * The objects of this class are immutable. + */ + public static class MultipleLinearRandomRetry implements RetryPolicy { + /** Pairs of numRetries and sleepSeconds */ + public static class Pair { + final int numRetries; + final int sleepMillis; + + public Pair(final int numRetries, final int sleepMillis) { + if (numRetries < 0) { + throw new IllegalArgumentException("numRetries = " + numRetries+" < 0"); + } + if (sleepMillis < 0) { + throw new IllegalArgumentException("sleepMillis = " + sleepMillis + " < 0"); + } + + this.numRetries = numRetries; + this.sleepMillis = sleepMillis; + } + + @Override + public String toString() { + return numRetries + "x" + sleepMillis + "ms"; + } + } + + private final List pairs; + private String myString; + + public MultipleLinearRandomRetry(List pairs) { + if (pairs == null || pairs.isEmpty()) { + throw new IllegalArgumentException("pairs must be neither null nor empty."); + } + this.pairs = Collections.unmodifiableList(pairs); + } + + @Override + public RetryAction shouldRetry(Exception e, int curRetry, int failovers, + boolean isMethodIdempotent) throws Exception { + final Pair p = searchPair(curRetry); + if (p == null) { + //no more retries. + return RetryAction.FAIL; + } + + //calculate sleep time and return. + final double ratio = RANDOM.get().nextDouble() + 0.5;//0.5 <= ratio <=1.5 + final long sleepTime = Math.round(p.sleepMillis * ratio); + return new RetryAction(RetryAction.RetryDecision.RETRY, sleepTime); + } + + /** + * Given the current number of retry, search the corresponding pair. + * @return the corresponding pair, + * or null if the current number of retry > maximum number of retry. + */ + private Pair searchPair(int curRetry) { + int i = 0; + for(; i < pairs.size() && curRetry > pairs.get(i).numRetries; i++) { + curRetry -= pairs.get(i).numRetries; + } + return i == pairs.size()? null: pairs.get(i); + } + + @Override + public int hashCode() { + return toString().hashCode(); + } + + @Override + public boolean equals(final Object that) { + if (this == that) { + return true; + } else if (that == null || this.getClass() != that.getClass()) { + return false; + } + return this.toString().equals(that.toString()); + } + + @Override + public String toString() { + if (myString == null) { + myString = getClass().getSimpleName() + pairs; + } + return myString; + } + + /** + * Parse the given string as a MultipleLinearRandomRetry object. + * The format of the string is "t_1, n_1, t_2, n_2, ...", + * where t_i and n_i are the i-th pair of sleep time and number of retires. + * Note that the white spaces in the string are ignored. + * + * @return the parsed object, or null if the parsing fails. + */ + public static MultipleLinearRandomRetry parseCommaSeparatedString(String s) { + final String[] elements = s.split(","); + if (elements.length == 0) { + LOG.warn("Illegal value: there is no element in \"" + s + "\"."); + return null; + } + if (elements.length % 2 != 0) { + LOG.warn("Illegal value: the number of elements in \"" + s + "\" is " + + elements.length + " but an even number of elements is expected."); + return null; + } + + final List pairs + = new ArrayList(); + + for(int i = 0; i < elements.length; ) { + //parse the i-th sleep-time + final int sleep = parsePositiveInt(elements, i++, s); + if (sleep == -1) { + return null; //parse fails + } + + //parse the i-th number-of-retries + final int retries = parsePositiveInt(elements, i++, s); + if (retries == -1) { + return null; //parse fails + } + + pairs.add(new RetryPolicies.MultipleLinearRandomRetry.Pair(retries, sleep)); + } + return new RetryPolicies.MultipleLinearRandomRetry(pairs); + } + + /** + * Parse the i-th element as an integer. + * @return -1 if the parsing fails or the parsed value <= 0; + * otherwise, return the parsed value. + */ + private static int parsePositiveInt(final String[] elements, + final int i, final String originalString) { + final String s = elements[i].trim(); + final int n; + try { + n = Integer.parseInt(s); + } catch(NumberFormatException nfe) { + LOG.warn("Failed to parse \"" + s + "\", which is the index " + i + + " element in \"" + originalString + "\"", nfe); + return -1; + } + + if (n <= 0) { + LOG.warn("The value " + n + " <= 0: it is parsed from the string \"" + + s + "\" which is the index " + i + " element in \"" + + originalString + "\""); + return -1; + } + return n; + } + } + static class ExceptionDependentRetry implements RetryPolicy { RetryPolicy defaultPolicy; @@ -265,6 +476,14 @@ static class ExponentialBackoffRetry extends RetryLimited { public ExponentialBackoffRetry( int maxRetries, long sleepTime, TimeUnit timeUnit) { super(maxRetries, sleepTime, timeUnit); + + if (maxRetries < 0) { + throw new IllegalArgumentException("maxRetries = " + maxRetries + " < 0"); + } else if (maxRetries >= Long.SIZE - 1) { + //calculateSleepTime may overflow. + throw new IllegalArgumentException("maxRetries = " + maxRetries + + " >= " + (Long.SIZE - 1)); + } } @Override @@ -353,11 +572,10 @@ public RetryAction shouldRetry(Exception e, int retries, * @param cap value at which to cap the base sleep time * @return an amount of time to sleep */ - @VisibleForTesting - public static long calculateExponentialTime(long time, int retries, + private static long calculateExponentialTime(long time, int retries, long cap) { - long baseTime = Math.min(time * ((long)1 << retries), cap); - return (long) (baseTime * (RAND.nextFloat() + 0.5)); + long baseTime = Math.min(time * (1L << retries), cap); + return (long) (baseTime * (RANDOM.get().nextDouble() + 0.5)); } private static long calculateExponentialTime(long time, int retries) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicy.java index ed673e950f..e1f3899457 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicy.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicy.java @@ -60,6 +60,12 @@ public RetryAction(RetryDecision action, long delayTime, String reason) { this.reason = reason; } + @Override + public String toString() { + return getClass().getSimpleName() + "(action=" + action + + ", delayMillis=" + delayMillis + ", reason=" + reason + ")"; + } + public enum RetryDecision { FAIL, RETRY, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java index 13e8a41eba..3cc6a2ec2d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java @@ -75,9 +75,10 @@ public static Object create(Class iface, FailoverProxyProvider proxyProvider, */ public static Object create(Class iface, Object implementation, Map methodNameToPolicyMap) { - return RetryProxy.create(iface, + return create(iface, new DefaultFailoverProxyProvider(iface, implementation), - methodNameToPolicyMap); + methodNameToPolicyMap, + RetryPolicies.TRY_ONCE_THEN_FAIL); } /** @@ -92,11 +93,13 @@ public static Object create(Class iface, Object implementation, * @return the retry proxy */ public static Object create(Class iface, FailoverProxyProvider proxyProvider, - Map methodNameToPolicyMap) { + Map methodNameToPolicyMap, + RetryPolicy defaultPolicy) { return Proxy.newProxyInstance( proxyProvider.getInterface().getClassLoader(), new Class[] { iface }, - new RetryInvocationHandler(proxyProvider, methodNameToPolicyMap) + new RetryInvocationHandler(proxyProvider, defaultPolicy, + methodNameToPolicyMap) ); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index ef32cfde3a..d382c99f61 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -18,47 +18,51 @@ package org.apache.hadoop.ipc; -import java.net.InetAddress; -import java.net.Socket; -import java.net.InetSocketAddress; -import java.net.SocketTimeoutException; -import java.net.UnknownHostException; -import java.io.IOException; -import java.io.DataInputStream; -import java.io.DataOutputStream; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.FilterInputStream; +import java.io.IOException; import java.io.InputStream; +import java.io.InterruptedIOException; import java.io.OutputStream; - +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; import java.security.PrivilegedExceptionAction; import java.util.Hashtable; import java.util.Iterator; +import java.util.Map.Entry; import java.util.Random; import java.util.Set; -import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import javax.net.SocketFactory; -import org.apache.commons.logging.*; - +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.io.retry.RetryPolicies; +import org.apache.hadoop.io.retry.RetryPolicy; +import org.apache.hadoop.io.retry.RetryPolicy.RetryAction; import org.apache.hadoop.ipc.protobuf.IpcConnectionContextProtos.IpcConnectionContextProto; import org.apache.hadoop.ipc.protobuf.RpcPayloadHeaderProtos.RpcPayloadHeaderProto; import org.apache.hadoop.ipc.protobuf.RpcPayloadHeaderProtos.RpcPayloadOperationProto; import org.apache.hadoop.ipc.protobuf.RpcPayloadHeaderProtos.RpcResponseHeaderProto; import org.apache.hadoop.ipc.protobuf.RpcPayloadHeaderProtos.RpcStatusProto; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableUtils; -import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.security.SaslRpcClient; @@ -67,8 +71,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.token.TokenSelector; import org.apache.hadoop.security.token.TokenInfo; +import org.apache.hadoop.security.token.TokenSelector; import org.apache.hadoop.util.ProtoUtil; import org.apache.hadoop.util.ReflectionUtils; @@ -80,8 +84,8 @@ */ public class Client { - public static final Log LOG = - LogFactory.getLog(Client.class); + public static final Log LOG = LogFactory.getLog(Client.class); + private Hashtable connections = new Hashtable(); @@ -228,8 +232,7 @@ private class Connection extends Thread { private int rpcTimeout; private int maxIdleTime; //connections will be culled if it was idle for //maxIdleTime msecs - private int maxRetries; //the max. no. of retries for socket connections - // the max. no. of retries for socket connections on time out exceptions + private final RetryPolicy connectionRetryPolicy; private int maxRetriesOnSocketTimeouts; private boolean tcpNoDelay; // if T then disable Nagle's Algorithm private boolean doPing; //do we need to send ping message @@ -253,7 +256,7 @@ public Connection(ConnectionId remoteId) throws IOException { } this.rpcTimeout = remoteId.getRpcTimeout(); this.maxIdleTime = remoteId.getMaxIdleTime(); - this.maxRetries = remoteId.getMaxRetries(); + this.connectionRetryPolicy = remoteId.connectionRetryPolicy; this.maxRetriesOnSocketTimeouts = remoteId.getMaxRetriesOnSocketTimeouts(); this.tcpNoDelay = remoteId.getTcpNoDelay(); this.doPing = remoteId.getDoPing(); @@ -488,7 +491,7 @@ private synchronized void setupConnection() throws IOException { if (updateAddress()) { timeoutFailures = ioFailures = 0; } - handleConnectionFailure(ioFailures++, maxRetries, ie); + handleConnectionFailure(ioFailures++, ie); } } } @@ -680,8 +683,36 @@ private void handleConnectionFailure( Thread.sleep(1000); } catch (InterruptedException ignored) {} - LOG.info("Retrying connect to server: " + server + - ". Already tried " + curRetries + " time(s)."); + LOG.info("Retrying connect to server: " + server + ". Already tried " + + curRetries + " time(s); maxRetries=" + maxRetries); + } + + private void handleConnectionFailure(int curRetries, IOException ioe + ) throws IOException { + closeConnection(); + + final RetryAction action; + try { + action = connectionRetryPolicy.shouldRetry(ioe, curRetries, 0, true); + } catch(Exception e) { + throw e instanceof IOException? (IOException)e: new IOException(e); + } + if (action.action == RetryAction.RetryDecision.FAIL) { + if (action.reason != null) { + LOG.warn("Failed to connect to server: " + server + ": " + + action.reason, ioe); + } + throw ioe; + } + + try { + Thread.sleep(action.delayMillis); + } catch (InterruptedException e) { + throw (IOException)new InterruptedIOException("Interrupted: action=" + + action + ", retry policy=" + connectionRetryPolicy).initCause(e); + } + LOG.info("Retrying connect to server: " + server + ". Already tried " + + curRetries + " time(s); retry policy is " + connectionRetryPolicy); } /** @@ -849,6 +880,10 @@ private void receiveResponse() { try { RpcResponseHeaderProto response = RpcResponseHeaderProto.parseDelimitedFrom(in); + if (response == null) { + throw new IOException("Response is null."); + } + int callId = response.getCallId(); if (LOG.isDebugEnabled()) LOG.debug(getName() + " got value #" + callId); @@ -1287,7 +1322,7 @@ public static class ConnectionId { private final String serverPrincipal; private final int maxIdleTime; //connections will be culled if it was idle for //maxIdleTime msecs - private final int maxRetries; //the max. no. of retries for socket connections + private final RetryPolicy connectionRetryPolicy; // the max. no. of retries for socket connections on time out exceptions private final int maxRetriesOnSocketTimeouts; private final boolean tcpNoDelay; // if T then disable Nagle's Algorithm @@ -1297,7 +1332,7 @@ public static class ConnectionId { ConnectionId(InetSocketAddress address, Class protocol, UserGroupInformation ticket, int rpcTimeout, String serverPrincipal, int maxIdleTime, - int maxRetries, int maxRetriesOnSocketTimeouts, + RetryPolicy connectionRetryPolicy, int maxRetriesOnSocketTimeouts, boolean tcpNoDelay, boolean doPing, int pingInterval) { this.protocol = protocol; this.address = address; @@ -1305,7 +1340,7 @@ public static class ConnectionId { this.rpcTimeout = rpcTimeout; this.serverPrincipal = serverPrincipal; this.maxIdleTime = maxIdleTime; - this.maxRetries = maxRetries; + this.connectionRetryPolicy = connectionRetryPolicy; this.maxRetriesOnSocketTimeouts = maxRetriesOnSocketTimeouts; this.tcpNoDelay = tcpNoDelay; this.doPing = doPing; @@ -1336,10 +1371,6 @@ int getMaxIdleTime() { return maxIdleTime; } - int getMaxRetries() { - return maxRetries; - } - /** max connection retries on socket time outs */ public int getMaxRetriesOnSocketTimeouts() { return maxRetriesOnSocketTimeouts; @@ -1357,6 +1388,12 @@ int getPingInterval() { return pingInterval; } + static ConnectionId getConnectionId(InetSocketAddress addr, + Class protocol, UserGroupInformation ticket, int rpcTimeout, + Configuration conf) throws IOException { + return getConnectionId(addr, protocol, ticket, rpcTimeout, null, conf); + } + /** * Returns a ConnectionId object. * @param addr Remote address for the connection. @@ -1367,9 +1404,18 @@ int getPingInterval() { * @return A ConnectionId instance * @throws IOException */ - public static ConnectionId getConnectionId(InetSocketAddress addr, + static ConnectionId getConnectionId(InetSocketAddress addr, Class protocol, UserGroupInformation ticket, int rpcTimeout, - Configuration conf) throws IOException { + RetryPolicy connectionRetryPolicy, Configuration conf) throws IOException { + + if (connectionRetryPolicy == null) { + final int max = conf.getInt( + CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, + CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_DEFAULT); + connectionRetryPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( + max, 1, TimeUnit.SECONDS); + } + String remotePrincipal = getRemotePrincipal(conf, addr, protocol); boolean doPing = conf.getBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, true); @@ -1377,8 +1423,7 @@ public static ConnectionId getConnectionId(InetSocketAddress addr, rpcTimeout, remotePrincipal, conf.getInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_DEFAULT), - conf.getInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, - CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_DEFAULT), + connectionRetryPolicy, conf.getInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT), @@ -1421,7 +1466,7 @@ public boolean equals(Object obj) { return isEqual(this.address, that.address) && this.doPing == that.doPing && this.maxIdleTime == that.maxIdleTime - && this.maxRetries == that.maxRetries + && isEqual(this.connectionRetryPolicy, that.connectionRetryPolicy) && this.pingInterval == that.pingInterval && isEqual(this.protocol, that.protocol) && this.rpcTimeout == that.rpcTimeout @@ -1434,11 +1479,10 @@ && isEqual(this.serverPrincipal, that.serverPrincipal) @Override public int hashCode() { - int result = 1; + int result = connectionRetryPolicy.hashCode(); result = PRIME * result + ((address == null) ? 0 : address.hashCode()); result = PRIME * result + (doPing ? 1231 : 1237); result = PRIME * result + maxIdleTime; - result = PRIME * result + maxRetries; result = PRIME * result + pingInterval; result = PRIME * result + ((protocol == null) ? 0 : protocol.hashCode()); result = PRIME * result + rpcTimeout; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java index 1338419a17..d355a85d4f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java @@ -36,9 +36,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataOutputOutputStream; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.ipc.Client.ConnectionId; import org.apache.hadoop.ipc.RPC.RpcInvoker; - import org.apache.hadoop.ipc.protobuf.HadoopRpcProtos.HadoopRpcRequestProto; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager; @@ -66,15 +66,24 @@ public class ProtobufRpcEngine implements RpcEngine { private static final ClientCache CLIENTS = new ClientCache(); + public ProtocolProxy getProxy(Class protocol, long clientVersion, + InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, + SocketFactory factory, int rpcTimeout) throws IOException { + return getProxy(protocol, clientVersion, addr, ticket, conf, factory, + rpcTimeout, null); + } + @Override @SuppressWarnings("unchecked") public ProtocolProxy getProxy(Class protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, - SocketFactory factory, int rpcTimeout) throws IOException { + SocketFactory factory, int rpcTimeout, RetryPolicy connectionRetryPolicy + ) throws IOException { - return new ProtocolProxy(protocol, (T) Proxy.newProxyInstance(protocol - .getClassLoader(), new Class[] { protocol }, new Invoker(protocol, - addr, ticket, conf, factory, rpcTimeout)), false); + final Invoker invoker = new Invoker(protocol, addr, ticket, conf, factory, + rpcTimeout, connectionRetryPolicy); + return new ProtocolProxy(protocol, (T) Proxy.newProxyInstance( + protocol.getClassLoader(), new Class[]{protocol}, invoker), false); } @Override @@ -97,11 +106,12 @@ private static class Invoker implements RpcInvocationHandler { private final long clientProtocolVersion; private final String protocolName; - public Invoker(Class protocol, InetSocketAddress addr, + private Invoker(Class protocol, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory, - int rpcTimeout) throws IOException { - this(protocol, Client.ConnectionId.getConnectionId(addr, protocol, - ticket, rpcTimeout, conf), conf, factory); + int rpcTimeout, RetryPolicy connectionRetryPolicy) throws IOException { + this(protocol, Client.ConnectionId.getConnectionId( + addr, protocol, ticket, rpcTimeout, connectionRetryPolicy, conf), + conf, factory); } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java index 56fbd7d5a1..6a8a71f83a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java @@ -41,6 +41,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.io.*; +import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.ipc.Client.ConnectionId; import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolInfoService; import org.apache.hadoop.net.NetUtils; @@ -326,7 +327,7 @@ public static ProtocolProxy waitForProtocolProxy(Class protocol, long clientVersion, InetSocketAddress addr, Configuration conf, long connTimeout) throws IOException { - return waitForProtocolProxy(protocol, clientVersion, addr, conf, 0, connTimeout); + return waitForProtocolProxy(protocol, clientVersion, addr, conf, 0, null, connTimeout); } /** @@ -347,7 +348,7 @@ public static T waitForProxy(Class protocol, int rpcTimeout, long timeout) throws IOException { return waitForProtocolProxy(protocol, clientVersion, addr, - conf, rpcTimeout, timeout).getProxy(); + conf, rpcTimeout, null, timeout).getProxy(); } /** @@ -367,6 +368,7 @@ public static ProtocolProxy waitForProtocolProxy(Class protocol, long clientVersion, InetSocketAddress addr, Configuration conf, int rpcTimeout, + RetryPolicy connectionRetryPolicy, long timeout) throws IOException { long startTime = System.currentTimeMillis(); IOException ioe; @@ -374,7 +376,7 @@ public static ProtocolProxy waitForProtocolProxy(Class protocol, try { return getProtocolProxy(protocol, clientVersion, addr, UserGroupInformation.getCurrentUser(), conf, NetUtils - .getDefaultSocketFactory(conf), rpcTimeout); + .getDefaultSocketFactory(conf), rpcTimeout, connectionRetryPolicy); } catch(ConnectException se) { // namenode has not been started LOG.info("Server at " + addr + " not available yet, Zzzzz..."); ioe = se; @@ -463,7 +465,7 @@ public static ProtocolProxy getProtocolProxy(Class protocol, Configuration conf, SocketFactory factory) throws IOException { return getProtocolProxy( - protocol, clientVersion, addr, ticket, conf, factory, 0); + protocol, clientVersion, addr, ticket, conf, factory, 0, null); } /** @@ -489,7 +491,7 @@ public static T getProxy(Class protocol, SocketFactory factory, int rpcTimeout) throws IOException { return getProtocolProxy(protocol, clientVersion, addr, ticket, - conf, factory, rpcTimeout).getProxy(); + conf, factory, rpcTimeout, null).getProxy(); } /** @@ -512,12 +514,13 @@ public static ProtocolProxy getProtocolProxy(Class protocol, UserGroupInformation ticket, Configuration conf, SocketFactory factory, - int rpcTimeout) throws IOException { + int rpcTimeout, + RetryPolicy connectionRetryPolicy) throws IOException { if (UserGroupInformation.isSecurityEnabled()) { SaslRpcServer.init(conf); } - return getProtocolEngine(protocol,conf).getProxy(protocol, - clientVersion, addr, ticket, conf, factory, rpcTimeout); + return getProtocolEngine(protocol,conf).getProxy(protocol, clientVersion, + addr, ticket, conf, factory, rpcTimeout, connectionRetryPolicy); } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java index d431b4a898..f74aa881d2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java @@ -97,8 +97,9 @@ public static RemoteException valueOf(Attributes attrs) { return new RemoteException(attrs.getValue("class"), attrs.getValue("message")); } - + + @Override public String toString() { - return className + ": " + getMessage(); + return getClass().getName() + "(" + className + "): " + getMessage(); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java index 09980da452..5dc48adef2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java @@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.ipc.Client.ConnectionId; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager; @@ -40,7 +41,8 @@ public interface RpcEngine { ProtocolProxy getProxy(Class protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, - SocketFactory factory, int rpcTimeout) throws IOException; + SocketFactory factory, int rpcTimeout, + RetryPolicy connectionRetryPolicy) throws IOException; /** Expert: Make multiple, parallel calls to a set of servers. */ Object[] call(Method method, Object[][] params, InetSocketAddress[] addrs, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java index 2ebf42a9aa..f61f0f2fd7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java @@ -31,6 +31,7 @@ import org.apache.commons.logging.*; import org.apache.hadoop.io.*; +import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.ipc.Client.ConnectionId; import org.apache.hadoop.ipc.RPC.RpcInvoker; import org.apache.hadoop.ipc.VersionedProtocol; @@ -259,9 +260,14 @@ static Client getClient(Configuration conf) { public ProtocolProxy getProxy(Class protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory, - int rpcTimeout) + int rpcTimeout, RetryPolicy connectionRetryPolicy) throws IOException { + if (connectionRetryPolicy != null) { + throw new UnsupportedOperationException( + "Not supported: connectionRetryPolicy=" + connectionRetryPolicy); + } + T proxy = (T) Proxy.newProxyInstance(protocol.getClassLoader(), new Class[] { protocol }, new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout)); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java index cc0c5c9f54..5d3d335b32 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java @@ -18,50 +18,55 @@ package org.apache.hadoop.ipc; +import static org.apache.hadoop.test.MetricsAsserts.assertCounter; +import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.io.Closeable; import java.io.IOException; -import java.net.ConnectException; -import java.net.InetAddress; -import java.net.InetSocketAddress; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; +import java.net.ConnectException; +import java.net.InetAddress; +import java.net.InetSocketAddress; import java.util.Arrays; import javax.net.SocketFactory; -import org.apache.commons.logging.*; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.io.UTF8; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.retry.RetryPolicies; +import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryProxy; import org.apache.hadoop.ipc.Client.ConnectionId; -import org.apache.hadoop.ipc.TestSaslRPC.TestSaslImpl; -import org.apache.hadoop.ipc.TestSaslRPC.TestSaslProtocol; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.authorize.Service; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.AccessControlException; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.MockitoUtil; import org.junit.Test; -import static org.junit.Assert.*; import com.google.protobuf.DescriptorProtos; import com.google.protobuf.DescriptorProtos.EnumDescriptorProto; -import static org.apache.hadoop.test.MetricsAsserts.*; - /** Unit tests for RPC. */ @SuppressWarnings("deprecation") public class TestRPC { @@ -250,7 +255,8 @@ public Object[] call(Method method, Object[][] params, InetSocketAddress[] addrs @Override public ProtocolProxy getProxy(Class protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, - SocketFactory factory, int rpcTimeout) throws IOException { + SocketFactory factory, int rpcTimeout, RetryPolicy connectionRetryPolicy + ) throws IOException { T proxy = (T) Proxy.newProxyInstance(protocol.getClassLoader(), new Class[] { protocol }, new StoppedInvocationHandler()); return new ProtocolProxy(protocol, proxy, false); diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5432479fb6..65e57f5fed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -222,6 +222,10 @@ Branch-2 ( Unreleased changes ) HDFS-3520. Add transfer rate logging to TransferFsImage. (eli) + HDFS-3504. Support configurable retry policy in DFSClient for RPC + connections and RPC calls, and add MultipleLinearRandomRetry, a new retry + policy. (szetszwo) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 309a72bb2e..51af55850d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -38,6 +38,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_STREAM_BUFFER_SIZE_DEFAULT = 4096; public static final String DFS_BYTES_PER_CHECKSUM_KEY = "dfs.bytes-per-checksum"; public static final int DFS_BYTES_PER_CHECKSUM_DEFAULT = 512; + public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY = "dfs.client.retry.policy.enabled"; + public static final boolean DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = false; + public static final String DFS_CLIENT_RETRY_POLICY_SPEC_KEY = "dfs.client.retry.policy.spec"; + public static final String DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT = "10000,6,60000,10"; //t1,n1,t2,n2,... public static final String DFS_CHECKSUM_TYPE_KEY = "dfs.checksum.type"; public static final String DFS_CHECKSUM_TYPE_DEFAULT = "CRC32C"; public static final String DFS_CLIENT_WRITE_PACKET_SIZE_KEY = "dfs.client-write-packet-size"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java index 27eddeb6e5..cc6517daa5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java @@ -47,10 +47,12 @@ import org.apache.hadoop.hdfs.protocolPB.RefreshUserMappingsProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.protocolPB.RefreshUserMappingsProtocolPB; import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.namenode.SafeModeException; import org.apache.hadoop.hdfs.server.protocol.JournalProtocol; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.retry.DefaultFailoverProxyProvider; import org.apache.hadoop.io.retry.FailoverProxyProvider; import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; @@ -66,6 +68,7 @@ import org.apache.hadoop.tools.GetUserMappingsProtocol; import com.google.common.base.Preconditions; +import com.google.protobuf.ServiceException; /** * Create proxy objects to communicate with a remote NN. All remote access to an @@ -240,12 +243,106 @@ private static NamenodeProtocol createNNProxyWithNamenodeProtocol( return new NamenodeProtocolTranslatorPB(proxy); } + /** + * Return the default retry policy used in RPC. + * + * If dfs.client.retry.policy.enabled == false, use TRY_ONCE_THEN_FAIL. + * + * Otherwise, first unwrap ServiceException if possible, and then + * (1) use multipleLinearRandomRetry for + * - SafeModeException, or + * - IOException other than RemoteException, or + * - ServiceException; and + * (2) use TRY_ONCE_THEN_FAIL for + * - non-SafeMode RemoteException, or + * - non-IOException. + * + * Note that dfs.client.retry.max < 0 is not allowed. + */ + private static RetryPolicy getDefaultRpcRetryPolicy(Configuration conf) { + final RetryPolicy multipleLinearRandomRetry = getMultipleLinearRandomRetry(conf); + if (LOG.isDebugEnabled()) { + LOG.debug("multipleLinearRandomRetry = " + multipleLinearRandomRetry); + } + if (multipleLinearRandomRetry == null) { + //no retry + return RetryPolicies.TRY_ONCE_THEN_FAIL; + } else { + return new RetryPolicy() { + @Override + public RetryAction shouldRetry(Exception e, int retries, int failovers, + boolean isMethodIdempotent) throws Exception { + if (e instanceof ServiceException) { + //unwrap ServiceException + final Throwable cause = e.getCause(); + if (cause != null && cause instanceof Exception) { + e = (Exception)cause; + } + } + + //see (1) and (2) in the javadoc of this method. + final RetryPolicy p; + if (e instanceof RemoteException) { + final RemoteException re = (RemoteException)e; + p = SafeModeException.class.getName().equals(re.getClassName())? + multipleLinearRandomRetry: RetryPolicies.TRY_ONCE_THEN_FAIL; + } else if (e instanceof IOException || e instanceof ServiceException) { + p = multipleLinearRandomRetry; + } else { //non-IOException + p = RetryPolicies.TRY_ONCE_THEN_FAIL; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("RETRY " + retries + ") policy=" + + p.getClass().getSimpleName() + ", exception=" + e); + } + LOG.info("RETRY " + retries + ") policy=" + + p.getClass().getSimpleName() + ", exception=" + e); + return p.shouldRetry(e, retries, failovers, isMethodIdempotent); + } + }; + } + } + + /** + * Return the MultipleLinearRandomRetry policy specified in the conf, + * or null if the feature is disabled. + * If the policy is specified in the conf but the policy cannot be parsed, + * the default policy is returned. + * + * Conf property: N pairs of sleep-time and number-of-retries + * dfs.client.retry.policy = "s1,n1,s2,n2,..." + */ + private static RetryPolicy getMultipleLinearRandomRetry(Configuration conf) { + final boolean enabled = conf.getBoolean( + DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, + DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT); + if (!enabled) { + return null; + } + + final String policy = conf.get( + DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY, + DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT); + + final RetryPolicy r = RetryPolicies.MultipleLinearRandomRetry.parseCommaSeparatedString(policy); + return r != null? r: RetryPolicies.MultipleLinearRandomRetry.parseCommaSeparatedString( + DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT); + } + private static ClientProtocol createNNProxyWithClientProtocol( InetSocketAddress address, Configuration conf, UserGroupInformation ugi, boolean withRetries) throws IOException { - ClientNamenodeProtocolPB proxy = (ClientNamenodeProtocolPB) NameNodeProxies - .createNameNodeProxy(address, conf, ugi, ClientNamenodeProtocolPB.class, 0); + RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class); + + final RetryPolicy defaultPolicy = getDefaultRpcRetryPolicy(conf); + final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class); + ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy( + ClientNamenodeProtocolPB.class, version, address, ugi, conf, + NetUtils.getDefaultSocketFactory(conf), 0, defaultPolicy).getProxy(); + if (withRetries) { // create the proxy with retries + RetryPolicy createPolicy = RetryPolicies .retryUpToMaximumCountWithFixedSleep(5, HdfsConstants.LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS); @@ -258,17 +355,21 @@ private static ClientProtocol createNNProxyWithClientProtocol( Map, RetryPolicy> exceptionToPolicyMap = new HashMap, RetryPolicy>(); exceptionToPolicyMap.put(RemoteException.class, RetryPolicies - .retryByRemoteException(RetryPolicies.TRY_ONCE_THEN_FAIL, + .retryByRemoteException(defaultPolicy, remoteExceptionToPolicyMap)); RetryPolicy methodPolicy = RetryPolicies.retryByException( - RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); + defaultPolicy, exceptionToPolicyMap); Map methodNameToPolicyMap = new HashMap(); methodNameToPolicyMap.put("create", methodPolicy); - proxy = (ClientNamenodeProtocolPB) RetryProxy - .create(ClientNamenodeProtocolPB.class, proxy, methodNameToPolicyMap); + proxy = (ClientNamenodeProtocolPB) RetryProxy.create( + ClientNamenodeProtocolPB.class, + new DefaultFailoverProxyProvider( + ClientNamenodeProtocolPB.class, proxy), + methodNameToPolicyMap, + defaultPolicy); } return new ClientNamenodeProtocolTranslatorPB(proxy); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 5c9c465997..65a67e59ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -25,8 +25,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY; @@ -39,6 +37,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI; @@ -66,12 +66,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.ha.HAServiceProtocol; +import org.apache.hadoop.ha.HAServiceProtocol.RequestSource; import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo; -import org.apache.hadoop.ha.HAServiceProtocolHelper; import org.apache.hadoop.ha.ServiceFailedException; -import org.apache.hadoop.ha.HAServiceProtocol.RequestSource; -import org.apache.hadoop.ha.protocolPB.HAServiceProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; @@ -1401,7 +1398,6 @@ public synchronized void restartNameNode(int nnIndex, boolean waitActive) waitClusterUp(); LOG.info("Restarted the namenode"); waitActive(); - LOG.info("Cluster is active"); } } @@ -1777,6 +1773,7 @@ public void waitActive() throws IOException { } } } + LOG.info("Cluster is active"); } private synchronized boolean shouldWait(DatanodeInfo[] dnInfo, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java index 2ba4dde0dd..4d8244dde7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java @@ -25,46 +25,53 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -import java.net.SocketTimeoutException; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.LongWritable; +import java.io.FileNotFoundException; import java.io.IOException; -import java.net.InetSocketAddress; import java.io.InputStream; import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.SocketTimeoutException; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.TimeUnit; import junit.framework.TestCase; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; +import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; -import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.retry.RetryPolicies.MultipleLinearRandomRetry; import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.log4j.Level; import org.mockito.Mockito; import org.mockito.internal.stubbing.answers.ThrowsException; import org.mockito.invocation.InvocationOnMock; @@ -341,7 +348,7 @@ public LocatedBlock answer(InvocationOnMock invocation) throws Throwable { // We shouldn't have gained an extra block by the RPC. assertEquals(blockCount, blockCount2); - return (LocatedBlock) ret2; + return ret2; } }).when(spyNN).addBlock(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any()); @@ -798,5 +805,149 @@ public void testRetryOnChecksumFailure() cluster.shutdown(); } } -} + /** Test client retry with namenode restarting. */ + public void testNamenodeRestart() throws Exception { + ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL); + + final List exceptions = new ArrayList(); + + final Path dir = new Path("/testNamenodeRestart"); + + final Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, true); + + final short numDatanodes = 3; + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(numDatanodes) + .build(); + try { + cluster.waitActive(); + + //create a file + final DistributedFileSystem dfs = cluster.getFileSystem(); + final long length = 1L << 20; + final Path file1 = new Path(dir, "foo"); + DFSTestUtil.createFile(dfs, file1, length, numDatanodes, 20120406L); + + //get file status + final FileStatus s1 = dfs.getFileStatus(file1); + assertEquals(length, s1.getLen()); + + //shutdown namenode + cluster.shutdownNameNode(0); + + //namenode is down, create another file in a thread + final Path file3 = new Path(dir, "file"); + final Thread thread = new Thread(new Runnable() { + @Override + public void run() { + try { + //it should retry till namenode is up. + final FileSystem fs = AppendTestUtil.createHdfsWithDifferentUsername(conf); + DFSTestUtil.createFile(fs, file3, length, numDatanodes, 20120406L); + } catch (Exception e) { + exceptions.add(e); + } + } + }); + thread.start(); + + //restart namenode in a new thread + new Thread(new Runnable() { + @Override + public void run() { + try { + //sleep, restart, and then wait active + TimeUnit.SECONDS.sleep(30); + cluster.restartNameNode(0, false); + cluster.waitActive(); + } catch (Exception e) { + exceptions.add(e); + } + } + }).start(); + + //namenode is down, it should retry until namenode is up again. + final FileStatus s2 = dfs.getFileStatus(file1); + assertEquals(s1, s2); + + //check file1 and file3 + thread.join(); + assertEquals(dfs.getFileChecksum(file1), dfs.getFileChecksum(file3)); + + //enter safe mode + dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + + //leave safe mode in a new thread + new Thread(new Runnable() { + @Override + public void run() { + try { + //sleep and then leave safe mode + TimeUnit.SECONDS.sleep(30); + dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + } catch (Exception e) { + exceptions.add(e); + } + } + }).start(); + + //namenode is in safe mode, create should retry until it leaves safe mode. + final Path file2 = new Path(dir, "bar"); + DFSTestUtil.createFile(dfs, file2, length, numDatanodes, 20120406L); + assertEquals(dfs.getFileChecksum(file1), dfs.getFileChecksum(file2)); + + //make sure it won't retry on exceptions like FileNotFoundException + final Path nonExisting = new Path(dir, "nonExisting"); + LOG.info("setPermission: " + nonExisting); + try { + dfs.setPermission(nonExisting, new FsPermission((short)0)); + fail(); + } catch(FileNotFoundException fnfe) { + LOG.info("GOOD!", fnfe); + } + + if (!exceptions.isEmpty()) { + LOG.error("There are " + exceptions.size() + " exception(s):"); + for(int i = 0; i < exceptions.size(); i++) { + LOG.error("Exception " + i, exceptions.get(i)); + } + fail(); + } + } finally { + cluster.shutdown(); + } + } + + public void testMultipleLinearRandomRetry() { + parseMultipleLinearRandomRetry(null, ""); + parseMultipleLinearRandomRetry(null, "11"); + parseMultipleLinearRandomRetry(null, "11,22,33"); + parseMultipleLinearRandomRetry(null, "11,22,33,44,55"); + parseMultipleLinearRandomRetry(null, "AA"); + parseMultipleLinearRandomRetry(null, "11,AA"); + parseMultipleLinearRandomRetry(null, "11,22,33,FF"); + parseMultipleLinearRandomRetry(null, "11,-22"); + parseMultipleLinearRandomRetry(null, "-11,22"); + + parseMultipleLinearRandomRetry("[22x11ms]", + "11,22"); + parseMultipleLinearRandomRetry("[22x11ms, 44x33ms]", + "11,22,33,44"); + parseMultipleLinearRandomRetry("[22x11ms, 44x33ms, 66x55ms]", + "11,22,33,44,55,66"); + parseMultipleLinearRandomRetry("[22x11ms, 44x33ms, 66x55ms]", + " 11, 22, 33, 44, 55, 66 "); + } + + static void parseMultipleLinearRandomRetry(String expected, String s) { + final MultipleLinearRandomRetry r = MultipleLinearRandomRetry.parseCommaSeparatedString(s); + LOG.info("input=" + s + ", parsed=" + r + ", expected=" + expected); + if (r == null) { + assertEquals(expected, null); + } else { + assertEquals("MultipleLinearRandomRetry" + expected, r.toString()); + } + } +} From e876b8119849667f79d1efd68c07184f740528a5 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Tue, 12 Jun 2012 06:22:06 +0000 Subject: [PATCH 31/91] Revert HADOOP-8491 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349133 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 9 ++-- .../java/org/apache/hadoop/io/IOUtils.java | 34 -------------- .../org/apache/hadoop/io/TestIOUtils.java | 44 +------------------ .../namenode/EditLogFileOutputStream.java | 4 +- 4 files changed, 6 insertions(+), 85 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 58f39e5aae..7b9332d954 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -242,12 +242,6 @@ Branch-2 ( Unreleased changes ) HADOOP-8485. Don't hardcode "Apache Hadoop 0.23" in the docs. (eli) - HADOOP-8488. test-patch.sh gives +1 even if the native build fails. - (Colin Patrick McCabe via eli) - - HADOOP-8491. Check for short writes when using FileChannel#write - and related methods. (Colin Patrick McCabe via eli) - BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active @@ -280,6 +274,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8405. ZKFC tests leak ZK instances. (todd) + HADOOP-8488. test-patch.sh gives +1 even if the native build fails. + (Colin Patrick McCabe via eli) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java index 65d8855f14..f5875d8d96 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java @@ -20,9 +20,6 @@ import java.io.*; import java.net.Socket; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.channels.WritableByteChannel; import org.apache.commons.logging.Log; @@ -248,35 +245,4 @@ public void write(byte[] b, int off, int len) throws IOException { public void write(int b) throws IOException { } } - - /** - * Write a ByteBuffer to a WritableByteChannel, handling short writes. - * - * @param bc The WritableByteChannel to write to. - * @param buf The input buffer - * @param offset The offset in the file to start writing at. - * @throws IOException On I/O error. - */ - public static void writeFully(WritableByteChannel bc, ByteBuffer buf) - throws IOException { - do { - bc.write(buf); - } while (buf.remaining() > 0); - } - - /** - * Write a ByteBuffer to a FileChannel at a given offset, - * handling short writes. - * - * @param fc The FileChannel to write to. - * @param buf The input buffer - * @param offset The offset in the file to start writing at. - * @throws IOException On I/O error. - */ - public static void writeFully(FileChannel fc, ByteBuffer buf, - long offset) throws IOException { - do { - offset += fc.write(buf, offset); - } while (buf.remaining() > 0); - } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java index 60c0703abc..d4f5057f7c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java @@ -21,13 +21,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import org.junit.Test; import org.mockito.Mockito; @@ -36,8 +32,7 @@ * Test cases for IOUtils.java */ public class TestIOUtils { - private static final String TEST_FILE_NAME = "test_file"; - + @Test public void testCopyBytesShouldCloseStreamsWhenCloseIsTrue() throws Exception { InputStream inputStream = Mockito.mock(InputStream.class); @@ -115,41 +110,4 @@ public void testCopyBytesWithCountShouldThrowOutTheStreamClosureExceptions() Mockito.verify(outputStream, Mockito.atLeastOnce()).close(); } - @Test - public void testWriteFully() throws IOException { - final int INPUT_BUFFER_LEN = 10000; - final int HALFWAY = 1 + (INPUT_BUFFER_LEN / 2); - byte[] input = new byte[INPUT_BUFFER_LEN]; - for (int i = 0; i < input.length; i++) { - input[i] = (byte)(i & 0xff); - } - byte[] output = new byte[input.length]; - - try { - RandomAccessFile raf = new RandomAccessFile(TEST_FILE_NAME, "rw"); - FileChannel fc = raf.getChannel(); - ByteBuffer buf = ByteBuffer.wrap(input); - IOUtils.writeFully(fc, buf); - raf.seek(0); - raf.read(output); - for (int i = 0; i < input.length; i++) { - assertEquals(input[i], output[i]); - } - buf.rewind(); - IOUtils.writeFully(fc, buf, HALFWAY); - for (int i = 0; i < HALFWAY; i++) { - assertEquals(input[i], output[i]); - } - raf.seek(0); - raf.read(output); - for (int i = HALFWAY; i < input.length; i++) { - assertEquals(input[i - HALFWAY], output[i]); - } - } finally { - File f = new File(TEST_FILE_NAME); - if (f.exists()) { - f.delete(); - } - } - } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java index 08a560ce12..dd8102ee74 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java @@ -206,10 +206,10 @@ private void preallocate() throws IOException { + fc.size()); } fill.position(0); - IOUtils.writeFully(fc, fill, position); + int written = fc.write(fill, position); if(FSNamesystem.LOG.isDebugEnabled()) { FSNamesystem.LOG.debug("Edit log size is now " + fc.size() + - " written " + fill.capacity() + " bytes " + " at offset " + position); + " written " + written + " bytes " + " at offset " + position); } } } From 75dd902c0e469a799ef00368963117ad8c02a096 Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Tue, 12 Jun 2012 17:48:16 +0000 Subject: [PATCH 32/91] HDFS-3408. BKJM : Namenode format fails, if there is no BK root. Contributed by Rakesh R. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349456 13f79535-47bb-0310-9956-ffa450edef68 --- .../bkjournal/BookKeeperJournalManager.java | 54 ++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java index 5317a0f66c..d6f19635f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java @@ -28,6 +28,7 @@ import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.util.ZkUtils; import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.ZooKeeper; @@ -36,6 +37,7 @@ import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.AsyncCallback.StringCallback; import java.util.Collection; import java.util.Collections; @@ -124,6 +126,12 @@ public class BookKeeperJournalManager implements JournalManager { private static final String BKJM_EDIT_INPROGRESS = "inprogress_"; + public static final String BKJM_ZK_LEDGERS_AVAILABLE_PATH + = "dfs.namenode.bookkeeperjournal.zk.availablebookies"; + + public static final String BKJM_ZK_LEDGERS_AVAILABLE_PATH_DEFAULT + = "/ledgers/available"; + private ZooKeeper zkc; private final Configuration conf; private final BookKeeper bkc; @@ -196,7 +204,7 @@ public BookKeeperJournalManager(Configuration conf, URI uri) zkc.create(ledgerPath, new byte[] {'0'}, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } - + prepareBookKeeperEnv(); bkc = new BookKeeper(new ClientConfiguration(), zkc); } catch (KeeperException e) { @@ -210,6 +218,50 @@ public BookKeeperJournalManager(Configuration conf, URI uri) maxTxId = new MaxTxId(zkc, maxTxIdPath); } + /** + * Pre-creating bookkeeper metadata path in zookeeper. + */ + private void prepareBookKeeperEnv() throws IOException { + // create bookie available path in zookeeper if it doesn't exists + final String zkAvailablePath = conf.get(BKJM_ZK_LEDGERS_AVAILABLE_PATH, + BKJM_ZK_LEDGERS_AVAILABLE_PATH_DEFAULT); + final CountDownLatch zkPathLatch = new CountDownLatch(1); + + StringCallback callback = new StringCallback() { + @Override + public void processResult(int rc, String path, Object ctx, String name) { + if (KeeperException.Code.OK.intValue() == rc + || KeeperException.Code.NODEEXISTS.intValue() == rc) { + LOG.info("Successfully created bookie available path : " + + zkAvailablePath); + zkPathLatch.countDown(); + } else { + KeeperException.Code code = KeeperException.Code.get(rc); + LOG + .error("Error : " + + KeeperException.create(code, path).getMessage() + + ", failed to create bookie available path : " + + zkAvailablePath); + } + } + }; + ZkUtils.createFullPathOptimistic(zkc, zkAvailablePath, new byte[0], + Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, callback, null); + + try { + if (!zkPathLatch.await(zkc.getSessionTimeout(), TimeUnit.MILLISECONDS)) { + throw new IOException("Couldn't create bookie available path :" + + zkAvailablePath + ", timed out " + zkc.getSessionTimeout() + + " millis"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException( + "Interrupted when creating the bookie available path : " + + zkAvailablePath, e); + } + } + /** * Start a new log segment in a BookKeeper ledger. * First ensure that we have the write lock for this journal. From 3bbcab2929bbbb8444952d249987219f18458cfa Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Tue, 12 Jun 2012 17:58:49 +0000 Subject: [PATCH 33/91] HDFS-3408. BKJM : Namenode format fails, if there is no BK root. Contributed by Rakesh R. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349459 13f79535-47bb-0310-9956-ffa450edef68 --- .../TestBookKeeperConfiguration.java | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java new file mode 100644 index 0000000000..df788a27da --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java @@ -0,0 +1,160 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.contrib.bkjournal; + +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.bookkeeper.util.LocalBookKeeper; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZKUtil; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.server.NIOServerCnxnFactory; +import org.apache.zookeeper.server.ZooKeeperServer; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestBookKeeperConfiguration { + private static final Log LOG = LogFactory + .getLog(TestBookKeeperConfiguration.class); + private static final int ZK_SESSION_TIMEOUT = 5000; + private static final String HOSTPORT = "127.0.0.1:2181"; + private static final int CONNECTION_TIMEOUT = 30000; + private static NIOServerCnxnFactory serverFactory; + private static ZooKeeperServer zks; + private static ZooKeeper zkc; + private static int ZooKeeperDefaultPort = 2181; + private static File ZkTmpDir; + private BookKeeperJournalManager bkjm; + private static final String BK_ROOT_PATH = "/ledgers"; + + private static ZooKeeper connectZooKeeper(String ensemble) + throws IOException, KeeperException, InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + + ZooKeeper zkc = new ZooKeeper(HOSTPORT, ZK_SESSION_TIMEOUT, new Watcher() { + public void process(WatchedEvent event) { + if (event.getState() == Watcher.Event.KeeperState.SyncConnected) { + latch.countDown(); + } + } + }); + if (!latch.await(ZK_SESSION_TIMEOUT, TimeUnit.MILLISECONDS)) { + throw new IOException("Zookeeper took too long to connect"); + } + return zkc; + } + + @BeforeClass + public static void setupZooKeeper() throws Exception { + // create a ZooKeeper server(dataDir, dataLogDir, port) + LOG.info("Starting ZK server"); + ZkTmpDir = File.createTempFile("zookeeper", "test"); + ZkTmpDir.delete(); + ZkTmpDir.mkdir(); + + try { + zks = new ZooKeeperServer(ZkTmpDir, ZkTmpDir, ZooKeeperDefaultPort); + serverFactory = new NIOServerCnxnFactory(); + serverFactory.configure(new InetSocketAddress(ZooKeeperDefaultPort), 10); + serverFactory.startup(zks); + } catch (Exception e) { + LOG.error("Exception while instantiating ZooKeeper", e); + } + + boolean b = LocalBookKeeper.waitForServerUp(HOSTPORT, CONNECTION_TIMEOUT); + LOG.debug("ZooKeeper server up: " + b); + } + + @Before + public void setup() throws Exception { + zkc = connectZooKeeper(HOSTPORT); + try { + ZKUtil.deleteRecursive(zkc, BK_ROOT_PATH); + } catch (KeeperException.NoNodeException e) { + LOG.debug("Ignoring no node exception on cleanup", e); + } catch (Exception e) { + LOG.error("Exception when deleting bookie root path in zk", e); + } + } + + @After + public void teardown() throws Exception { + if (null != zkc) { + zkc.close(); + } + if (null != bkjm) { + bkjm.close(); + } + } + + @AfterClass + public static void teardownZooKeeper() throws Exception { + if (null != zkc) { + zkc.close(); + } + } + + /** + * Verify the BKJM is creating the bookie available path configured in + * 'dfs.namenode.bookkeeperjournal.zk.availablebookies' + */ + @Test + public void testWithConfiguringBKAvailablePath() throws Exception { + // set Bookie available path in the configuration + String bkAvailablePath + = BookKeeperJournalManager.BKJM_ZK_LEDGERS_AVAILABLE_PATH_DEFAULT; + Configuration conf = new Configuration(); + conf.setStrings(BookKeeperJournalManager.BKJM_ZK_LEDGERS_AVAILABLE_PATH, + bkAvailablePath); + Assert.assertNull(bkAvailablePath + " already exists", zkc.exists( + bkAvailablePath, false)); + bkjm = new BookKeeperJournalManager(conf, URI.create("bookkeeper://" + + HOSTPORT + "/hdfsjournal-WithBKPath")); + Assert.assertNotNull("Bookie available path : " + bkAvailablePath + + " doesn't exists", zkc.exists(bkAvailablePath, false)); + } + + /** + * Verify the BKJM is creating the bookie available default path, when there + * is no 'dfs.namenode.bookkeeperjournal.zk.availablebookies' configured + */ + @Test + public void testDefaultBKAvailablePath() throws Exception { + Configuration conf = new Configuration(); + Assert.assertNull(BK_ROOT_PATH + " already exists", zkc.exists( + BK_ROOT_PATH, false)); + new BookKeeperJournalManager(conf, URI.create("bookkeeper://" + HOSTPORT + + "/hdfsjournal-DefaultBKPath")); + Assert.assertNotNull("Bookie available path : " + BK_ROOT_PATH + + " doesn't exists", zkc.exists(BK_ROOT_PATH, false)); + } +} From 7fb6cf60aa0d9c0500d814c215f5327a356186b0 Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Tue, 12 Jun 2012 18:19:46 +0000 Subject: [PATCH 34/91] HDFS-3389. Document the BKJM usage in Namenode HA. Contributed by Uma Maheswara Rao G and Ivan Kelly. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349466 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/site/apt/HDFSHighAvailability.apt.vm | 153 +++++++++++++++++- 1 file changed, 152 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm index 67d423221c..7e7cb66772 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm @@ -712,4 +712,155 @@ digest:hdfs-zkfcs:vlUvLnd8MlacsE80rDuu6ONESbM=:rwcda Even if automatic failover is configured, you may initiate a manual failover using the same <<>> command. It will perform a coordinated - failover. \ No newline at end of file + failover. + + +* BookKeeper as a Shared storage (EXPERIMENTAL) + + One option for shared storage for the NameNode is BookKeeper. + BookKeeper achieves high availability and strong durability guarantees by replicating + edit log entries across multiple storage nodes. The edit log can be striped across + the storage nodes for high performance. Fencing is supported in the protocol, i.e, + BookKeeper will not allow two writers to write the single edit log. + + The meta data for BookKeeper is stored in ZooKeeper. + In current HA architecture, a Zookeeper cluster is required for ZKFC. The same cluster can be + for BookKeeper metadata. + + For more details on building a BookKeeper cluster, please refer to the + {{{http://zookeeper.apache.org/bookkeeper/docs/trunk/bookkeeperConfig.html }BookKeeper documentation}} + + The BookKeeperJournalManager is an implementation of the HDFS JournalManager interface, which allows custom write ahead logging implementations to be plugged into the HDFS NameNode. + + **<> + + To use BookKeeperJournalManager, add the following to hdfs-site.xml. + +---- + + dfs.namenode.shared.edits.dir + bookkeeper://zk1:2181;zk2:2181;zk3:2181/hdfsjournal + + + + dfs.namenode.edits.journal-plugin.bookkeeper + org.apache.hadoop.contrib.bkjournal.BookKeeperJournalManager + +---- + + The URI format for bookkeeper is <<>> is a list of semi-colon separated, zookeeper host:port + pairs. In the example above there are 3 servers, in the ensemble, + zk1, zk2 & zk3, each one listening on port 2181. + + <<<[root znode]>>> is the path of the zookeeper znode, under which the edit log + information will be stored. + + The class specified for the journal-plugin must be available in the NameNode's + classpath. We explain how to generate a jar file with the journal manager and + its dependencies, and how to put it into the classpath below. + + *** <> + + * <> - + Number of bytes a bookkeeper journal stream will buffer before + forcing a flush. Default is 1024. + +---- + + dfs.namenode.bookkeeperjournal.output-buffer-size + 1024 + +---- + + * <> - + Number of bookkeeper servers in edit log ensembles. This + is the number of bookkeeper servers which need to be available + for the edit log to be writable. Default is 3. + +---- + + dfs.namenode.bookkeeperjournal.ensemble-size + 3 + +---- + + * <> - + Number of bookkeeper servers in the write quorum. This is the + number of bookkeeper servers which must have acknowledged the + write of an entry before it is considered written. Default is 2. + +---- + + dfs.namenode.bookkeeperjournal.quorum-size + 2 + +---- + + * <> - + Password to use when creating edit log segments. + +---- + + dfs.namenode.bookkeeperjournal.digestPw + myPassword + +---- + + * <> - + Session timeout for Zookeeper client from BookKeeper Journal Manager. + Hadoop recommends that this value should be less than the ZKFC + session timeout value. Default value is 3000. + +---- + + dfs.namenode.bookkeeperjournal.zk.session.timeout + 3000 + +---- + + *** <> + + To generate the distribution packages for BK journal, do the + following. + + $ mvn clean package -Pdist + + This will generate a jar with the BookKeeperJournalManager, all the dependencies + needed by the journal manager, + hadoop-hdfs/src/contrib/bkjournal/target/hadoop-hdfs-bkjournal-.jar + + Note that the -Pdist part of the build command is important, as otherwise + the dependencies would not be packaged in the jar. The dependencies included in + the jar are {{{http://maven.apache.org/plugins/maven-shade-plugin/}shaded}} to + avoid conflicts with other dependencies of the NameNode. + + *** <> + + To run a HDFS namenode using BookKeeper as a backend, copy the bkjournal + jar, generated above, into the lib directory of hdfs. In the standard + distribution of HDFS, this is at $HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/ + + cp hadoop-hdfs/src/contrib/bkjournal/target/hadoop-hdfs-bkjournal-.jar $HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/ + + *** <> + + 1) NameNode format command will not format the BookKeeper data automatically. + We have to clean the data manually from BookKeeper cluster + and create the /ledgers/available path in Zookeeper. +---- +$ zkCli.sh create /ledgers 0 +$ zkCli.sh create /ledgers/available 0 +---- + Note: + bookkeeper://zk1:2181;zk2:2181;zk3:2181/hdfsjournal + The final part /hdfsjournal specifies the znode in zookeeper where + ledger metadata will be stored. Administrators may set this to anything + they wish. + + 2) Security in BookKeeper. BookKeeper does not support SASL nor SSL for + connections between the NameNode and BookKeeper storage nodes. + + 3) Auto-Recovery of storage node failures. Work inprogress + {{{https://issues.apache.org/jira/browse/BOOKKEEPER-237 }BOOKKEEPER-237}}. + Currently we have the tools to manually recover the data from failed storage nodes. \ No newline at end of file From 4a70e982808c409fad67b1d9821d363651c2011e Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Tue, 12 Jun 2012 20:45:57 +0000 Subject: [PATCH 35/91] HADOOP-8458. Add management hook to AuthenticationHandler to enable delegation token operations support (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349514 13f79535-47bb-0310-9956-ffa450edef68 --- .../server/AuthenticationFilter.java | 70 +++++----- .../server/AuthenticationHandler.java | 28 ++++ .../server/KerberosAuthenticationHandler.java | 21 +++ .../server/PseudoAuthenticationHandler.java | 21 +++ .../server/TestAuthenticationFilter.java | 127 ++++++++++++++++-- .../hadoop-common/CHANGES.txt | 3 + 6 files changed, 223 insertions(+), 47 deletions(-) diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index 28a4d3de90..b1795d1524 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -341,45 +341,49 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha LOG.warn("AuthenticationToken ignored: " + ex.getMessage()); token = null; } - if (token == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Request [{}] triggering authentication", getRequestURL(httpRequest)); - } - token = authHandler.authenticate(httpRequest, httpResponse); - if (token != null && token != AuthenticationToken.ANONYMOUS) { - token.setExpires(System.currentTimeMillis() + getValidity() * 1000); - } - newToken = true; - } - if (token != null) { - unauthorizedResponse = false; - if (LOG.isDebugEnabled()) { - LOG.debug("Request [{}] user [{}] authenticated", getRequestURL(httpRequest), token.getUserName()); + if (authHandler.managementOperation(token, httpRequest, httpResponse)) { + if (token == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Request [{}] triggering authentication", getRequestURL(httpRequest)); + } + token = authHandler.authenticate(httpRequest, httpResponse); + if (token != null && token != AuthenticationToken.ANONYMOUS) { + token.setExpires(System.currentTimeMillis() + getValidity() * 1000); + } + newToken = true; } - final AuthenticationToken authToken = token; - httpRequest = new HttpServletRequestWrapper(httpRequest) { - - @Override - public String getAuthType() { - return authToken.getType(); + if (token != null) { + unauthorizedResponse = false; + if (LOG.isDebugEnabled()) { + LOG.debug("Request [{}] user [{}] authenticated", getRequestURL(httpRequest), token.getUserName()); } + final AuthenticationToken authToken = token; + httpRequest = new HttpServletRequestWrapper(httpRequest) { - @Override - public String getRemoteUser() { - return authToken.getUserName(); - } + @Override + public String getAuthType() { + return authToken.getType(); + } - @Override - public Principal getUserPrincipal() { - return (authToken != AuthenticationToken.ANONYMOUS) ? authToken : null; + @Override + public String getRemoteUser() { + return authToken.getUserName(); + } + + @Override + public Principal getUserPrincipal() { + return (authToken != AuthenticationToken.ANONYMOUS) ? authToken : null; + } + }; + if (newToken && token != AuthenticationToken.ANONYMOUS) { + String signedToken = signer.sign(token.toString()); + Cookie cookie = createCookie(signedToken); + httpResponse.addCookie(cookie); } - }; - if (newToken && token != AuthenticationToken.ANONYMOUS) { - String signedToken = signer.sign(token.toString()); - Cookie cookie = createCookie(signedToken); - httpResponse.addCookie(cookie); + filterChain.doFilter(httpRequest, httpResponse); } - filterChain.doFilter(httpRequest, httpResponse); + } else { + unauthorizedResponse = false; } } catch (AuthenticationException ex) { unauthorizedMsg = ex.toString(); diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java index 958680fcad..7cafe8bcbd 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java @@ -58,6 +58,34 @@ public interface AuthenticationHandler { */ public void destroy(); + /** + * Performs an authentication management operation. + *

+ * This is useful for handling operations like get/renew/cancel + * delegation tokens which are being handled as operations of the + * service end-point. + *

+ * If the method returns TRUE the request will continue normal + * processing, this means the method has not produced any HTTP response. + *

+ * If the method returns FALSE the request will end, this means + * the method has produced the corresponding HTTP response. + * + * @param token the authentication token if any, otherwise NULL. + * @param request the HTTP client request. + * @param response the HTTP client response. + * @return TRUE if the request should be processed as a regular + * request, + * FALSE otherwise. + * + * @throws IOException thrown if an IO error occurred. + * @throws AuthenticationException thrown if an Authentication error occurred. + */ + public boolean managementOperation(AuthenticationToken token, + HttpServletRequest request, + HttpServletResponse response) + throws IOException, AuthenticationException; + /** * Performs an authentication step for the given HTTP client request. *

diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java index 8cad2cc220..07b64f48f9 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java @@ -232,6 +232,27 @@ protected String getKeytab() { return keytab; } + /** + * This is an empty implementation, it always returns TRUE. + * + * + * + * @param token the authentication token if any, otherwise NULL. + * @param request the HTTP client request. + * @param response the HTTP client response. + * + * @return TRUE + * @throws IOException it is never thrown. + * @throws AuthenticationException it is never thrown. + */ + @Override + public boolean managementOperation(AuthenticationToken token, + HttpServletRequest request, + HttpServletResponse response) + throws IOException, AuthenticationException { + return true; + } + /** * It enforces the the Kerberos SPNEGO authentication sequence returning an {@link AuthenticationToken} only * after the Kerberos SPNEGO sequence has completed successfully. diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java index 336c36e4d2..1a2f98c1c9 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java @@ -93,6 +93,27 @@ public String getType() { return TYPE; } + /** + * This is an empty implementation, it always returns TRUE. + * + * + * + * @param token the authentication token if any, otherwise NULL. + * @param request the HTTP client request. + * @param response the HTTP client response. + * + * @return TRUE + * @throws IOException it is never thrown. + * @throws AuthenticationException it is never thrown. + */ + @Override + public boolean managementOperation(AuthenticationToken token, + HttpServletRequest request, + HttpServletResponse response) + throws IOException, AuthenticationException { + return true; + } + /** * Authenticates an HTTP client request. *

diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java index 4f1bc111a7..746cf7b089 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java @@ -71,6 +71,7 @@ public void testInitEmpty() throws Exception { public static class DummyAuthenticationHandler implements AuthenticationHandler { public static boolean init; + public static boolean managementOperationReturn; public static boolean destroy; public static final String TYPE = "dummy"; @@ -83,6 +84,19 @@ public static void reset() { @Override public void init(Properties config) throws ServletException { init = true; + managementOperationReturn = + config.getProperty("management.operation.return", "true").equals("true"); + } + + @Override + public boolean managementOperation(AuthenticationToken token, + HttpServletRequest request, + HttpServletResponse response) + throws IOException, AuthenticationException { + if (!managementOperationReturn) { + response.setStatus(HttpServletResponse.SC_ACCEPTED); + } + return managementOperationReturn; } @Override @@ -170,10 +184,14 @@ public void testInit() throws Exception { filter = new AuthenticationFilter(); try { FilterConfig config = Mockito.mock(FilterConfig.class); + Mockito.when(config.getInitParameter("management.operation.return")). + thenReturn("true"); Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn( DummyAuthenticationHandler.class.getName()); Mockito.when(config.getInitParameterNames()).thenReturn( - new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE)).elements()); + new Vector( + Arrays.asList(AuthenticationFilter.AUTH_TYPE, + "management.operation.return")).elements()); filter.init(config); assertTrue(DummyAuthenticationHandler.init); } finally { @@ -201,10 +219,14 @@ public void testGetRequestURL() throws Exception { AuthenticationFilter filter = new AuthenticationFilter(); try { FilterConfig config = Mockito.mock(FilterConfig.class); + Mockito.when(config.getInitParameter("management.operation.return")). + thenReturn("true"); Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn( DummyAuthenticationHandler.class.getName()); Mockito.when(config.getInitParameterNames()).thenReturn( - new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE)).elements()); + new Vector( + Arrays.asList(AuthenticationFilter.AUTH_TYPE, + "management.operation.return")).elements()); filter.init(config); HttpServletRequest request = Mockito.mock(HttpServletRequest.class); @@ -221,12 +243,16 @@ public void testGetToken() throws Exception { AuthenticationFilter filter = new AuthenticationFilter(); try { FilterConfig config = Mockito.mock(FilterConfig.class); + Mockito.when(config.getInitParameter("management.operation.return")). + thenReturn("true"); Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn( DummyAuthenticationHandler.class.getName()); Mockito.when(config.getInitParameter(AuthenticationFilter.SIGNATURE_SECRET)).thenReturn("secret"); Mockito.when(config.getInitParameterNames()).thenReturn( - new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE, - AuthenticationFilter.SIGNATURE_SECRET)).elements()); + new Vector( + Arrays.asList(AuthenticationFilter.AUTH_TYPE, + AuthenticationFilter.SIGNATURE_SECRET, + "management.operation.return")).elements()); filter.init(config); AuthenticationToken token = new AuthenticationToken("u", "p", DummyAuthenticationHandler.TYPE); @@ -250,12 +276,15 @@ public void testGetTokenExpired() throws Exception { AuthenticationFilter filter = new AuthenticationFilter(); try { FilterConfig config = Mockito.mock(FilterConfig.class); + Mockito.when(config.getInitParameter("management.operation.return")).thenReturn("true"); Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn( DummyAuthenticationHandler.class.getName()); Mockito.when(config.getInitParameter(AuthenticationFilter.SIGNATURE_SECRET)).thenReturn("secret"); Mockito.when(config.getInitParameterNames()).thenReturn( - new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE, - AuthenticationFilter.SIGNATURE_SECRET)).elements()); + new Vector( + Arrays.asList(AuthenticationFilter.AUTH_TYPE, + AuthenticationFilter.SIGNATURE_SECRET, + "management.operation.return")).elements()); filter.init(config); AuthenticationToken token = new AuthenticationToken("u", "p", "invalidtype"); @@ -284,12 +313,16 @@ public void testGetTokenInvalidType() throws Exception { AuthenticationFilter filter = new AuthenticationFilter(); try { FilterConfig config = Mockito.mock(FilterConfig.class); + Mockito.when(config.getInitParameter("management.operation.return")). + thenReturn("true"); Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn( DummyAuthenticationHandler.class.getName()); Mockito.when(config.getInitParameter(AuthenticationFilter.SIGNATURE_SECRET)).thenReturn("secret"); Mockito.when(config.getInitParameterNames()).thenReturn( - new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE, - AuthenticationFilter.SIGNATURE_SECRET)).elements()); + new Vector( + Arrays.asList(AuthenticationFilter.AUTH_TYPE, + AuthenticationFilter.SIGNATURE_SECRET, + "management.operation.return")).elements()); filter.init(config); AuthenticationToken token = new AuthenticationToken("u", "p", "invalidtype"); @@ -318,10 +351,14 @@ public void testDoFilterNotAuthenticated() throws Exception { AuthenticationFilter filter = new AuthenticationFilter(); try { FilterConfig config = Mockito.mock(FilterConfig.class); + Mockito.when(config.getInitParameter("management.operation.return")). + thenReturn("true"); Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn( DummyAuthenticationHandler.class.getName()); Mockito.when(config.getInitParameterNames()).thenReturn( - new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE)).elements()); + new Vector( + Arrays.asList(AuthenticationFilter.AUTH_TYPE, + "management.operation.return")).elements()); filter.init(config); HttpServletRequest request = Mockito.mock(HttpServletRequest.class); @@ -353,6 +390,8 @@ private void _testDoFilterAuthentication(boolean withDomainPath, boolean invalid AuthenticationFilter filter = new AuthenticationFilter(); try { FilterConfig config = Mockito.mock(FilterConfig.class); + Mockito.when(config.getInitParameter("management.operation.return")). + thenReturn("true"); Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn( DummyAuthenticationHandler.class.getName()); Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TOKEN_VALIDITY)).thenReturn("1000"); @@ -360,7 +399,8 @@ private void _testDoFilterAuthentication(boolean withDomainPath, boolean invalid Mockito.when(config.getInitParameterNames()).thenReturn( new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE, AuthenticationFilter.AUTH_TOKEN_VALIDITY, - AuthenticationFilter.SIGNATURE_SECRET)).elements()); + AuthenticationFilter.SIGNATURE_SECRET, + "management.operation.return")).elements()); if (withDomainPath) { Mockito.when(config.getInitParameter(AuthenticationFilter.COOKIE_DOMAIN)).thenReturn(".foo.com"); @@ -370,7 +410,8 @@ private void _testDoFilterAuthentication(boolean withDomainPath, boolean invalid AuthenticationFilter.AUTH_TOKEN_VALIDITY, AuthenticationFilter.SIGNATURE_SECRET, AuthenticationFilter.COOKIE_DOMAIN, - AuthenticationFilter.COOKIE_PATH)).elements()); + AuthenticationFilter.COOKIE_PATH, + "management.operation.return")).elements()); } filter.init(config); @@ -458,10 +499,14 @@ public void testDoFilterAuthenticated() throws Exception { AuthenticationFilter filter = new AuthenticationFilter(); try { FilterConfig config = Mockito.mock(FilterConfig.class); + Mockito.when(config.getInitParameter("management.operation.return")). + thenReturn("true"); Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn( DummyAuthenticationHandler.class.getName()); Mockito.when(config.getInitParameterNames()).thenReturn( - new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE)).elements()); + new Vector( + Arrays.asList(AuthenticationFilter.AUTH_TYPE, + "management.operation.return")).elements()); filter.init(config); HttpServletRequest request = Mockito.mock(HttpServletRequest.class); @@ -503,10 +548,14 @@ public void testDoFilterAuthenticatedExpired() throws Exception { AuthenticationFilter filter = new AuthenticationFilter(); try { FilterConfig config = Mockito.mock(FilterConfig.class); + Mockito.when(config.getInitParameter("management.operation.return")). + thenReturn("true"); Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn( DummyAuthenticationHandler.class.getName()); Mockito.when(config.getInitParameterNames()).thenReturn( - new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE)).elements()); + new Vector( + Arrays.asList(AuthenticationFilter.AUTH_TYPE, + "management.operation.return")).elements()); filter.init(config); HttpServletRequest request = Mockito.mock(HttpServletRequest.class); @@ -563,10 +612,14 @@ public void testDoFilterAuthenticatedInvalidType() throws Exception { AuthenticationFilter filter = new AuthenticationFilter(); try { FilterConfig config = Mockito.mock(FilterConfig.class); + Mockito.when(config.getInitParameter("management.operation.return")). + thenReturn("true"); Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn( DummyAuthenticationHandler.class.getName()); Mockito.when(config.getInitParameterNames()).thenReturn( - new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE)).elements()); + new Vector( + Arrays.asList(AuthenticationFilter.AUTH_TYPE, + "management.operation.return")).elements()); filter.init(config); HttpServletRequest request = Mockito.mock(HttpServletRequest.class); @@ -618,4 +671,50 @@ public Object answer(InvocationOnMock invocation) throws Throwable { } } + public void testManagementOperation() throws Exception { + AuthenticationFilter filter = new AuthenticationFilter(); + try { + FilterConfig config = Mockito.mock(FilterConfig.class); + Mockito.when(config.getInitParameter("management.operation.return")). + thenReturn("false"); + Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)). + thenReturn(DummyAuthenticationHandler.class.getName()); + Mockito.when(config.getInitParameterNames()).thenReturn( + new Vector( + Arrays.asList(AuthenticationFilter.AUTH_TYPE, + "management.operation.return")).elements()); + filter.init(config); + + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + Mockito.when(request.getRequestURL()). + thenReturn(new StringBuffer("http://foo:8080/bar")); + + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + + FilterChain chain = Mockito.mock(FilterChain.class); + + filter.doFilter(request, response, chain); + Mockito.verify(response).setStatus(HttpServletResponse.SC_ACCEPTED); + Mockito.verifyNoMoreInteractions(response); + + Mockito.reset(request); + Mockito.reset(response); + + AuthenticationToken token = new AuthenticationToken("u", "p", "t"); + token.setExpires(System.currentTimeMillis() + 1000); + Signer signer = new Signer("secret".getBytes()); + String tokenSigned = signer.sign(token.toString()); + Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned); + Mockito.when(request.getCookies()).thenReturn(new Cookie[]{cookie}); + + filter.doFilter(request, response, chain); + + Mockito.verify(response).setStatus(HttpServletResponse.SC_ACCEPTED); + Mockito.verifyNoMoreInteractions(response); + + } finally { + filter.destroy(); + } + } + } diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 7b9332d954..6b80bf8244 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -174,6 +174,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8135. Add ByteBufferReadable interface to FSDataInputStream. (Henry Robinson via atm) + HADOOP-8458. Add management hook to AuthenticationHandler to enable + delegation token operations support (tucu) + IMPROVEMENTS HADOOP-8340. SNAPSHOT build versions should compare as less than their eventual From 418b50cd2a1d7fd960b31c67e03fe39642fe18cc Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Tue, 12 Jun 2012 21:52:46 +0000 Subject: [PATCH 36/91] HDFS-3228. Move DelegationTokenRenewer to common (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349555 13f79535-47bb-0310-9956-ffa450edef68 --- .../java/org/apache/hadoop/fs}/DelegationTokenRenewer.java | 5 ++--- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java | 2 +- .../java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) rename {hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs}/DelegationTokenRenewer.java (98%) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenRenewer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java similarity index 98% rename from hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenRenewer.java rename to hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java index 349d71baeb..2d4975949d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenRenewer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.hdfs.security.token.delegation; +package org.apache.hadoop.fs; import java.io.IOException; import java.lang.ref.WeakReference; @@ -25,7 +25,6 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -161,4 +160,4 @@ public void run() { } } } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 65e57f5fed..25e9f1baa4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -344,6 +344,8 @@ Branch-2 ( Unreleased changes ) HDFS-3432. TestDFSZKFailoverController tries to fail over too early (todd) + HDFS-3228. Move DelegationTokenRenewer to common (tucu) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java index 989fc12300..cd6601b231 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java @@ -37,6 +37,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.DelegationTokenRenewer; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; @@ -46,7 +47,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.tools.DelegationTokenFetcher; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 0b355ccaa9..7dbaded8d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -39,6 +39,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.DelegationTokenRenewer; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileAlreadyExistsException; @@ -57,7 +58,6 @@ import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.UnresolvedPathException; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.namenode.SafeModeException; From 02b2f1707a637470ae7992f7843a882bb7d73b6c Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Tue, 12 Jun 2012 22:02:56 +0000 Subject: [PATCH 37/91] Amendment to my previous commit, fixing JIRA in CHANGES.txt from HDFS-3228 to HDFS-3428 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349556 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 25e9f1baa4..fb5ae88cd9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -344,7 +344,7 @@ Branch-2 ( Unreleased changes ) HDFS-3432. TestDFSZKFailoverController tries to fail over too early (todd) - HDFS-3228. Move DelegationTokenRenewer to common (tucu) + HDFS-3428. Move DelegationTokenRenewer to common (tucu) Release 2.0.0-alpha - 05-23-2012 From 15f7bc3dd634d82ee845f57a29d1ecb91fe044b8 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Tue, 12 Jun 2012 22:10:12 +0000 Subject: [PATCH 38/91] HADOOP-8507. Avoid OOM while deserializing DelegationTokenIdentifer. Contributed by Colin Patrick McCabe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349561 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../java/org/apache/hadoop/fs/FileStatus.java | 16 +++---- .../fs/permission/PermissionStatus.java | 8 ++-- .../main/java/org/apache/hadoop/io/Text.java | 32 ++++++++++++-- .../AbstractDelegationTokenIdentifier.java | 24 +++++++++-- .../java/org/apache/hadoop/io/TestText.java | 18 ++++---- .../token/delegation/TestDelegationToken.java | 43 +++++++++++++++++++ 7 files changed, 115 insertions(+), 29 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 6b80bf8244..ce5d814389 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -245,6 +245,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8485. Don't hardcode "Apache Hadoop 0.23" in the docs. (eli) + HADOOP-8507. Avoid OOM while deserializing DelegationTokenIdentifer. + (Colin Patrick McCabe via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java index 4cc2c182d5..2757475faf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java @@ -254,7 +254,7 @@ public void setSymlink(final Path p) { // Writable ////////////////////////////////////////////////// public void write(DataOutput out) throws IOException { - Text.writeString(out, getPath().toString(), Text.ONE_MEGABYTE); + Text.writeString(out, getPath().toString(), Text.DEFAULT_MAX_LEN); out.writeLong(getLen()); out.writeBoolean(isDirectory()); out.writeShort(getReplication()); @@ -262,16 +262,16 @@ public void write(DataOutput out) throws IOException { out.writeLong(getModificationTime()); out.writeLong(getAccessTime()); getPermission().write(out); - Text.writeString(out, getOwner(), Text.ONE_MEGABYTE); - Text.writeString(out, getGroup(), Text.ONE_MEGABYTE); + Text.writeString(out, getOwner(), Text.DEFAULT_MAX_LEN); + Text.writeString(out, getGroup(), Text.DEFAULT_MAX_LEN); out.writeBoolean(isSymlink()); if (isSymlink()) { - Text.writeString(out, getSymlink().toString(), Text.ONE_MEGABYTE); + Text.writeString(out, getSymlink().toString(), Text.DEFAULT_MAX_LEN); } } public void readFields(DataInput in) throws IOException { - String strPath = Text.readString(in, Text.ONE_MEGABYTE); + String strPath = Text.readString(in, Text.DEFAULT_MAX_LEN); this.path = new Path(strPath); this.length = in.readLong(); this.isdir = in.readBoolean(); @@ -280,10 +280,10 @@ public void readFields(DataInput in) throws IOException { modification_time = in.readLong(); access_time = in.readLong(); permission.readFields(in); - owner = Text.readString(in, Text.ONE_MEGABYTE); - group = Text.readString(in, Text.ONE_MEGABYTE); + owner = Text.readString(in, Text.DEFAULT_MAX_LEN); + group = Text.readString(in, Text.DEFAULT_MAX_LEN); if (in.readBoolean()) { - this.symlink = new Path(Text.readString(in, Text.ONE_MEGABYTE)); + this.symlink = new Path(Text.readString(in, Text.DEFAULT_MAX_LEN)); } else { this.symlink = null; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java index 5642d0f5b9..f47226f1e2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java @@ -84,8 +84,8 @@ public PermissionStatus applyUMask(FsPermission umask) { /** {@inheritDoc} */ public void readFields(DataInput in) throws IOException { - username = Text.readString(in, Text.ONE_MEGABYTE); - groupname = Text.readString(in, Text.ONE_MEGABYTE); + username = Text.readString(in, Text.DEFAULT_MAX_LEN); + groupname = Text.readString(in, Text.DEFAULT_MAX_LEN); permission = FsPermission.read(in); } @@ -110,8 +110,8 @@ public static void write(DataOutput out, String username, String groupname, FsPermission permission) throws IOException { - Text.writeString(out, username, Text.ONE_MEGABYTE); - Text.writeString(out, groupname, Text.ONE_MEGABYTE); + Text.writeString(out, username, Text.DEFAULT_MAX_LEN); + Text.writeString(out, groupname, Text.DEFAULT_MAX_LEN); permission.write(out); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java index 78748b05f8..a4f80ea886 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java @@ -287,6 +287,20 @@ public void readFields(DataInput in) throws IOException { in.readFully(bytes, 0, newLength); length = newLength; } + + public void readFields(DataInput in, int maxLength) throws IOException { + int newLength = WritableUtils.readVInt(in); + if (newLength < 0) { + throw new IOException("tried to deserialize " + newLength + + " bytes of data! newLength must be non-negative."); + } else if (newLength >= maxLength) { + throw new IOException("tried to deserialize " + newLength + + " bytes of data, but maxLength = " + maxLength); + } + setCapacity(newLength, false); + in.readFully(bytes, 0, newLength); + length = newLength; + } /** Skips over one Text in the input. */ public static void skip(DataInput in) throws IOException { @@ -304,6 +318,16 @@ public void write(DataOutput out) throws IOException { out.write(bytes, 0, length); } + public void write(DataOutput out, int maxLength) throws IOException { + if (length > maxLength) { + throw new IOException("data was too long to write! Expected " + + "less than or equal to " + maxLength + " bytes, but got " + + length + " bytes."); + } + WritableUtils.writeVInt(out, length); + out.write(bytes, 0, length); + } + /** Returns true iff o is a Text with the same contents. */ public boolean equals(Object o) { if (o instanceof Text) @@ -417,7 +441,7 @@ public static ByteBuffer encode(String string, boolean replace) return bytes; } - static final public int ONE_MEGABYTE = 1024 * 1024; + static final public int DEFAULT_MAX_LEN = 1024 * 1024; /** Read a UTF8 encoded string from in */ @@ -432,7 +456,7 @@ public static String readString(DataInput in) throws IOException { */ public static String readString(DataInput in, int maxLength) throws IOException { - int length = WritableUtils.readVIntInRange(in, 0, maxLength - 1); + int length = WritableUtils.readVIntInRange(in, 0, maxLength); byte [] bytes = new byte[length]; in.readFully(bytes, 0, length); return decode(bytes); @@ -454,9 +478,9 @@ public static int writeString(DataOutput out, String s, int maxLength) throws IOException { ByteBuffer bytes = encode(s); int length = bytes.limit(); - if (length >= maxLength) { + if (length > maxLength) { throw new IOException("string was too long to write! Expected " + - "less than " + maxLength + " bytes, but got " + + "less than or equal to " + maxLength + " bytes, but got " + length + " bytes."); } WritableUtils.writeVInt(out, length); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java index c7738086fb..8c3c1b2d35 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java @@ -31,6 +31,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.TokenIdentifier; +import com.google.common.annotations.VisibleForTesting; + @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) @InterfaceStability.Evolving public abstract class AbstractDelegationTokenIdentifier @@ -173,16 +175,17 @@ public void readFields(DataInput in) throws IOException { throw new IOException("Unknown version of delegation token " + version); } - owner.readFields(in); - renewer.readFields(in); - realUser.readFields(in); + owner.readFields(in, Text.DEFAULT_MAX_LEN); + renewer.readFields(in, Text.DEFAULT_MAX_LEN); + realUser.readFields(in, Text.DEFAULT_MAX_LEN); issueDate = WritableUtils.readVLong(in); maxDate = WritableUtils.readVLong(in); sequenceNumber = WritableUtils.readVInt(in); masterKeyId = WritableUtils.readVInt(in); } - public void write(DataOutput out) throws IOException { + @VisibleForTesting + void writeImpl(DataOutput out) throws IOException { out.writeByte(VERSION); owner.write(out); renewer.write(out); @@ -193,6 +196,19 @@ public void write(DataOutput out) throws IOException { WritableUtils.writeVInt(out, masterKeyId); } + public void write(DataOutput out) throws IOException { + if (owner.getLength() > Text.DEFAULT_MAX_LEN) { + throw new IOException("owner is too long to be serialized!"); + } + if (renewer.getLength() > Text.DEFAULT_MAX_LEN) { + throw new IOException("renewer is too long to be serialized!"); + } + if (realUser.getLength() > Text.DEFAULT_MAX_LEN) { + throw new IOException("realuser is too long to be serialized!"); + } + writeImpl(out); + } + public String toString() { StringBuilder buffer = new StringBuilder(); buffer diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java index 9bf83b906f..21da8c0dce 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java @@ -137,38 +137,38 @@ public void testIO() throws Exception { } } - public void doTestLimitedIO(String str, int strLen) throws IOException { + public void doTestLimitedIO(String str, int len) throws IOException { DataOutputBuffer out = new DataOutputBuffer(); DataInputBuffer in = new DataInputBuffer(); out.reset(); try { - Text.writeString(out, str, strLen); + Text.writeString(out, str, len); fail("expected writeString to fail when told to write a string " + "that was too long! The string was '" + str + "'"); } catch (IOException e) { } - Text.writeString(out, str, strLen + 1); + Text.writeString(out, str, len + 1); // test that it reads correctly in.reset(out.getData(), out.getLength()); - in.mark(strLen); + in.mark(len); String after; try { - after = Text.readString(in, strLen); + after = Text.readString(in, len); fail("expected readString to fail when told to read a string " + "that was too long! The string was '" + str + "'"); } catch (IOException e) { } in.reset(); - after = Text.readString(in, strLen + 1); + after = Text.readString(in, len + 1); assertTrue(str.equals(after)); } public void testLimitedIO() throws Exception { - doTestLimitedIO("abcd", 4); - doTestLimitedIO("", 0); - doTestLimitedIO("1", 1); + doTestLimitedIO("abcd", 3); + doTestLimitedIO("foo bar baz", 10); + doTestLimitedIO("1", 0); } public void testCompare() throws Exception { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java index e2388ad550..91268109bd 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java @@ -19,6 +19,7 @@ package org.apache.hadoop.security.token.delegation; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.DataInput; import java.io.DataInputStream; import java.io.DataOutput; @@ -387,4 +388,46 @@ public void testDelegationTokenNullRenewer() throws Exception { } } + private boolean testDelegationTokenIdentiferSerializationRoundTrip(Text owner, + Text renewer, Text realUser) throws IOException { + TestDelegationTokenIdentifier dtid = new TestDelegationTokenIdentifier( + owner, renewer, realUser); + DataOutputBuffer out = new DataOutputBuffer(); + dtid.writeImpl(out); + DataInputBuffer in = new DataInputBuffer(); + in.reset(out.getData(), out.getLength()); + try { + TestDelegationTokenIdentifier dtid2 = + new TestDelegationTokenIdentifier(); + dtid2.readFields(in); + assertTrue(dtid.equals(dtid2)); + return true; + } catch(IOException e){ + return false; + } + } + + @Test + public void testSimpleDtidSerialization() throws IOException { + assertTrue(testDelegationTokenIdentiferSerializationRoundTrip( + new Text("owner"), new Text("renewer"), new Text("realUser"))); + assertTrue(testDelegationTokenIdentiferSerializationRoundTrip( + new Text(""), new Text(""), new Text(""))); + assertTrue(testDelegationTokenIdentiferSerializationRoundTrip( + new Text(""), new Text("b"), new Text(""))); + } + + @Test + public void testOverlongDtidSerialization() throws IOException { + byte[] bigBuf = new byte[Text.DEFAULT_MAX_LEN + 1]; + for (int i = 0; i < bigBuf.length; i++) { + bigBuf[i] = 0; + } + assertFalse(testDelegationTokenIdentiferSerializationRoundTrip( + new Text(bigBuf), new Text("renewer"), new Text("realUser"))); + assertFalse(testDelegationTokenIdentiferSerializationRoundTrip( + new Text("owner"), new Text(bigBuf), new Text("realUser"))); + assertFalse(testDelegationTokenIdentiferSerializationRoundTrip( + new Text("owner"), new Text("renewer"), new Text(bigBuf))); + } } From 1d5c34710f44544429916c263e80ea6ef07027dc Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Tue, 12 Jun 2012 22:13:35 +0000 Subject: [PATCH 39/91] Fix bug in CHANGES.txt git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349565 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ce5d814389..b4f298560d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -245,6 +245,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8485. Don't hardcode "Apache Hadoop 0.23" in the docs. (eli) + HADOOP-8488. test-patch.sh gives +1 even if the native build fails. + (Colin Patrick McCabe via eli) + HADOOP-8507. Avoid OOM while deserializing DelegationTokenIdentifer. (Colin Patrick McCabe via eli) @@ -280,9 +283,6 @@ Branch-2 ( Unreleased changes ) HADOOP-8405. ZKFC tests leak ZK instances. (todd) - HADOOP-8488. test-patch.sh gives +1 even if the native build fails. - (Colin Patrick McCabe via eli) - Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES From 8948e9b15853d86227755ba0702bfc0093d7be44 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 13 Jun 2012 02:22:06 +0000 Subject: [PATCH 40/91] HADOOP-8465. hadoop-auth should support ephemeral authentication (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349613 13f79535-47bb-0310-9956-ffa450edef68 --- .../server/AuthenticationFilter.java | 5 +- .../server/AuthenticationToken.java | 10 +-- .../server/TestAuthenticationFilter.java | 67 ++++++++++++------- .../hadoop-common/CHANGES.txt | 2 + 4 files changed, 52 insertions(+), 32 deletions(-) diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index b1795d1524..0bd78f5901 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -347,7 +347,8 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha LOG.debug("Request [{}] triggering authentication", getRequestURL(httpRequest)); } token = authHandler.authenticate(httpRequest, httpResponse); - if (token != null && token != AuthenticationToken.ANONYMOUS) { + if (token != null && token.getExpires() != 0 && + token != AuthenticationToken.ANONYMOUS) { token.setExpires(System.currentTimeMillis() + getValidity() * 1000); } newToken = true; @@ -375,7 +376,7 @@ public Principal getUserPrincipal() { return (authToken != AuthenticationToken.ANONYMOUS) ? authToken : null; } }; - if (newToken && token != AuthenticationToken.ANONYMOUS) { + if (newToken && !token.isExpired() && token != AuthenticationToken.ANONYMOUS) { String signedToken = signer.sign(token.toString()); Cookie cookie = createCookie(signedToken); httpResponse.addCookie(cookie); diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationToken.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationToken.java index e2856a3e1f..ff68847c8a 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationToken.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationToken.java @@ -115,10 +115,10 @@ public void setExpires(long expires) { */ private void generateToken() { StringBuffer sb = new StringBuffer(); - sb.append(USER_NAME).append("=").append(userName).append(ATTR_SEPARATOR); - sb.append(PRINCIPAL).append("=").append(principal).append(ATTR_SEPARATOR); - sb.append(TYPE).append("=").append(type).append(ATTR_SEPARATOR); - sb.append(EXPIRES).append("=").append(expires); + sb.append(USER_NAME).append("=").append(getUserName()).append(ATTR_SEPARATOR); + sb.append(PRINCIPAL).append("=").append(getName()).append(ATTR_SEPARATOR); + sb.append(TYPE).append("=").append(getType()).append(ATTR_SEPARATOR); + sb.append(EXPIRES).append("=").append(getExpires()); token = sb.toString(); } @@ -165,7 +165,7 @@ public long getExpires() { * @return if the token has expired. */ public boolean isExpired() { - return expires != -1 && System.currentTimeMillis() > expires; + return getExpires() != -1 && System.currentTimeMillis() > getExpires(); } /** diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java index 746cf7b089..1c31e54ba5 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java @@ -73,6 +73,7 @@ public static class DummyAuthenticationHandler implements AuthenticationHandler public static boolean init; public static boolean managementOperationReturn; public static boolean destroy; + public static boolean expired; public static final String TYPE = "dummy"; @@ -86,6 +87,7 @@ public void init(Properties config) throws ServletException { init = true; managementOperationReturn = config.getProperty("management.operation.return", "true").equals("true"); + expired = config.getProperty("expired.token", "false").equals("true"); } @Override @@ -116,7 +118,7 @@ public AuthenticationToken authenticate(HttpServletRequest request, HttpServletR String param = request.getParameter("authenticated"); if (param != null && param.equals("true")) { token = new AuthenticationToken("u", "p", "t"); - token.setExpires(System.currentTimeMillis() + 1000); + token.setExpires((expired) ? 0 : System.currentTimeMillis() + 1000); } else { response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); } @@ -386,12 +388,16 @@ public Object answer(InvocationOnMock invocation) throws Throwable { } } - private void _testDoFilterAuthentication(boolean withDomainPath, boolean invalidToken) throws Exception { + private void _testDoFilterAuthentication(boolean withDomainPath, + boolean invalidToken, + boolean expired) throws Exception { AuthenticationFilter filter = new AuthenticationFilter(); try { FilterConfig config = Mockito.mock(FilterConfig.class); Mockito.when(config.getInitParameter("management.operation.return")). thenReturn("true"); + Mockito.when(config.getInitParameter("expired.token")). + thenReturn(Boolean.toString(expired)); Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn( DummyAuthenticationHandler.class.getName()); Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TOKEN_VALIDITY)).thenReturn("1000"); @@ -400,7 +406,8 @@ private void _testDoFilterAuthentication(boolean withDomainPath, boolean invalid new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE, AuthenticationFilter.AUTH_TOKEN_VALIDITY, AuthenticationFilter.SIGNATURE_SECRET, - "management.operation.return")).elements()); + "management.operation.return", + "expired.token")).elements()); if (withDomainPath) { Mockito.when(config.getInitParameter(AuthenticationFilter.COOKIE_DOMAIN)).thenReturn(".foo.com"); @@ -457,26 +464,32 @@ public Object answer(InvocationOnMock invocation) throws Throwable { filter.doFilter(request, response, chain); - assertNotNull(setCookie[0]); - assertEquals(AuthenticatedURL.AUTH_COOKIE, setCookie[0].getName()); - assertTrue(setCookie[0].getValue().contains("u=")); - assertTrue(setCookie[0].getValue().contains("p=")); - assertTrue(setCookie[0].getValue().contains("t=")); - assertTrue(setCookie[0].getValue().contains("e=")); - assertTrue(setCookie[0].getValue().contains("s=")); - assertTrue(calledDoFilter[0]); - - Signer signer = new Signer("secret".getBytes()); - String value = signer.verifyAndExtract(setCookie[0].getValue()); - AuthenticationToken token = AuthenticationToken.parse(value); - assertEquals(System.currentTimeMillis() + 1000 * 1000, token.getExpires(), 100); - - if (withDomainPath) { - assertEquals(".foo.com", setCookie[0].getDomain()); - assertEquals("/bar", setCookie[0].getPath()); + if (expired) { + Mockito.verify(response, Mockito.never()). + addCookie(Mockito.any(Cookie.class)); } else { - assertNull(setCookie[0].getDomain()); - assertNull(setCookie[0].getPath()); + assertNotNull(setCookie[0]); + assertEquals(AuthenticatedURL.AUTH_COOKIE, setCookie[0].getName()); + assertTrue(setCookie[0].getValue().contains("u=")); + assertTrue(setCookie[0].getValue().contains("p=")); + assertTrue(setCookie[0].getValue().contains("t=")); + assertTrue(setCookie[0].getValue().contains("e=")); + assertTrue(setCookie[0].getValue().contains("s=")); + assertTrue(calledDoFilter[0]); + + Signer signer = new Signer("secret".getBytes()); + String value = signer.verifyAndExtract(setCookie[0].getValue()); + AuthenticationToken token = AuthenticationToken.parse(value); + assertEquals(System.currentTimeMillis() + 1000 * 1000, + token.getExpires(), 100); + + if (withDomainPath) { + assertEquals(".foo.com", setCookie[0].getDomain()); + assertEquals("/bar", setCookie[0].getPath()); + } else { + assertNull(setCookie[0].getDomain()); + assertNull(setCookie[0].getPath()); + } } } finally { filter.destroy(); @@ -484,15 +497,19 @@ public Object answer(InvocationOnMock invocation) throws Throwable { } public void testDoFilterAuthentication() throws Exception { - _testDoFilterAuthentication(false, false); + _testDoFilterAuthentication(false, false, false); + } + + public void testDoFilterAuthenticationImmediateExpiration() throws Exception { + _testDoFilterAuthentication(false, false, true); } public void testDoFilterAuthenticationWithInvalidToken() throws Exception { - _testDoFilterAuthentication(false, true); + _testDoFilterAuthentication(false, true, false); } public void testDoFilterAuthenticationWithDomainPath() throws Exception { - _testDoFilterAuthentication(true, false); + _testDoFilterAuthentication(true, false, false); } public void testDoFilterAuthenticated() throws Exception { diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b4f298560d..aa28d7dea0 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -177,6 +177,8 @@ Branch-2 ( Unreleased changes ) HADOOP-8458. Add management hook to AuthenticationHandler to enable delegation token operations support (tucu) + HADOOP-8465. hadoop-auth should support ephemeral authentication (tucu) + IMPROVEMENTS HADOOP-8340. SNAPSHOT build versions should compare as less than their eventual From b33b9fdd3691cd25b0787c7046524a8ffcfc56cc Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Wed, 13 Jun 2012 02:42:52 +0000 Subject: [PATCH 41/91] HDFS-3531. EditLogFileOutputStream#preallocate should check for incomplete writes. Contributed by Colin Patrick McCabe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349616 13f79535-47bb-0310-9956-ffa450edef68 --- .../java/org/apache/hadoop/io/IOUtils.java | 33 ++++++++++++++ .../org/apache/hadoop/io/TestIOUtils.java | 44 ++++++++++++++++++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../namenode/EditLogFileOutputStream.java | 4 +- 4 files changed, 81 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java index f5875d8d96..6969d19043 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java @@ -20,6 +20,9 @@ import java.io.*; import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; import org.apache.commons.logging.Log; @@ -245,4 +248,34 @@ public void write(byte[] b, int off, int len) throws IOException { public void write(int b) throws IOException { } } + + /** + * Write a ByteBuffer to a WritableByteChannel, handling short writes. + * + * @param bc The WritableByteChannel to write to + * @param buf The input buffer + * @throws IOException On I/O error + */ + public static void writeFully(WritableByteChannel bc, ByteBuffer buf) + throws IOException { + do { + bc.write(buf); + } while (buf.remaining() > 0); + } + + /** + * Write a ByteBuffer to a FileChannel at a given offset, + * handling short writes. + * + * @param fc The FileChannel to write to + * @param buf The input buffer + * @param offset The offset in the file to start writing at + * @throws IOException On I/O error + */ + public static void writeFully(FileChannel fc, ByteBuffer buf, + long offset) throws IOException { + do { + offset += fc.write(buf, offset); + } while (buf.remaining() > 0); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java index d4f5057f7c..60c0703abc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java @@ -21,9 +21,13 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import org.junit.Test; import org.mockito.Mockito; @@ -32,7 +36,8 @@ * Test cases for IOUtils.java */ public class TestIOUtils { - + private static final String TEST_FILE_NAME = "test_file"; + @Test public void testCopyBytesShouldCloseStreamsWhenCloseIsTrue() throws Exception { InputStream inputStream = Mockito.mock(InputStream.class); @@ -110,4 +115,41 @@ public void testCopyBytesWithCountShouldThrowOutTheStreamClosureExceptions() Mockito.verify(outputStream, Mockito.atLeastOnce()).close(); } + @Test + public void testWriteFully() throws IOException { + final int INPUT_BUFFER_LEN = 10000; + final int HALFWAY = 1 + (INPUT_BUFFER_LEN / 2); + byte[] input = new byte[INPUT_BUFFER_LEN]; + for (int i = 0; i < input.length; i++) { + input[i] = (byte)(i & 0xff); + } + byte[] output = new byte[input.length]; + + try { + RandomAccessFile raf = new RandomAccessFile(TEST_FILE_NAME, "rw"); + FileChannel fc = raf.getChannel(); + ByteBuffer buf = ByteBuffer.wrap(input); + IOUtils.writeFully(fc, buf); + raf.seek(0); + raf.read(output); + for (int i = 0; i < input.length; i++) { + assertEquals(input[i], output[i]); + } + buf.rewind(); + IOUtils.writeFully(fc, buf, HALFWAY); + for (int i = 0; i < HALFWAY; i++) { + assertEquals(input[i], output[i]); + } + raf.seek(0); + raf.read(output); + for (int i = HALFWAY; i < input.length; i++) { + assertEquals(input[i - HALFWAY], output[i]); + } + } finally { + File f = new File(TEST_FILE_NAME); + if (f.exists()) { + f.delete(); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index fb5ae88cd9..155e45f83c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -328,6 +328,9 @@ Branch-2 ( Unreleased changes ) HDFS-3522. If a namenode is in safemode, it should throw SafeModeException when getBlockLocations has zero locations. (Brandon Li via szetszwo) + HDFS-3531. EditLogFileOutputStream#preallocate should check for + incomplete writes. (Colin Patrick McCabe via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java index dd8102ee74..08a560ce12 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java @@ -206,10 +206,10 @@ private void preallocate() throws IOException { + fc.size()); } fill.position(0); - int written = fc.write(fill, position); + IOUtils.writeFully(fc, fill, position); if(FSNamesystem.LOG.isDebugEnabled()) { FSNamesystem.LOG.debug("Edit log size is now " + fc.size() + - " written " + written + " bytes " + " at offset " + position); + " written " + fill.capacity() + " bytes " + " at offset " + position); } } } From a73b8b244251c0e8f757fe2ab12b34e9cee9b602 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Wed, 13 Jun 2012 03:31:47 +0000 Subject: [PATCH 42/91] HDFS-3372. offlineEditsViewer should be able to read a binary edits file with recovery mode. Contributed by Colin Patrick McCabe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349628 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../OfflineEditsBinaryLoader.java | 47 +++++++----- .../OfflineEditsLoader.java | 12 ++-- .../OfflineEditsViewer.java | 71 ++++++++++++++----- .../OfflineEditsXmlLoader.java | 16 ++--- .../TestOfflineEditsViewer.java | 64 ++++++++++++++--- 6 files changed, 153 insertions(+), 60 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 155e45f83c..450ba4a0ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -226,6 +226,9 @@ Branch-2 ( Unreleased changes ) connections and RPC calls, and add MultipleLinearRandomRetry, a new retry policy. (szetszwo) + HDFS-3372. offlineEditsViewer should be able to read a binary + edits file with recovery mode. (Colin Patrick McCabe via eli) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsBinaryLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsBinaryLoader.java index 969ecf684e..c35e6248a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsBinaryLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsBinaryLoader.java @@ -18,9 +18,12 @@ package org.apache.hadoop.hdfs.tools.offlineEditsViewer; import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; - +import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream; @@ -33,17 +36,21 @@ class OfflineEditsBinaryLoader implements OfflineEditsLoader { private OfflineEditsVisitor visitor; private EditLogInputStream inputStream; - private boolean fixTxIds; + private final boolean fixTxIds; + private final boolean recoveryMode; private long nextTxId; + public static final Log LOG = + LogFactory.getLog(OfflineEditsBinaryLoader.class.getName()); /** * Constructor */ public OfflineEditsBinaryLoader(OfflineEditsVisitor visitor, - EditLogInputStream inputStream) { + EditLogInputStream inputStream, OfflineEditsViewer.Flags flags) { this.visitor = visitor; this.inputStream = inputStream; - this.fixTxIds = false; + this.fixTxIds = flags.getFixTxIds(); + this.recoveryMode = flags.getRecoveryMode(); this.nextTxId = -1; } @@ -51,9 +58,9 @@ public OfflineEditsBinaryLoader(OfflineEditsVisitor visitor, * Loads edits file, uses visitor to process all elements */ public void loadEdits() throws IOException { - try { - visitor.start(inputStream.getVersion()); - while (true) { + visitor.start(inputStream.getVersion()); + while (true) { + try { FSEditLogOp op = inputStream.readOp(); if (op == null) break; @@ -68,16 +75,24 @@ public void loadEdits() throws IOException { nextTxId++; } visitor.visitOp(op); + } catch (IOException e) { + if (!recoveryMode) { + // Tell the visitor to clean up, then re-throw the exception + visitor.close(e); + throw e; + } + LOG.error("Got IOException while reading stream! Resyncing.", e); + inputStream.resync(); + } catch (RuntimeException e) { + if (!recoveryMode) { + // Tell the visitor to clean up, then re-throw the exception + visitor.close(e); + throw e; + } + LOG.error("Got RuntimeException while reading stream! Resyncing.", e); + inputStream.resync(); } - visitor.close(null); - } catch(IOException e) { - // Tell the visitor to clean up, then re-throw the exception - visitor.close(e); - throw e; } - } - - public void setFixTxIds() { - fixTxIds = true; + visitor.close(null); } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsLoader.java index a314352d4d..0ce1e78614 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsLoader.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream; @@ -36,13 +37,12 @@ interface OfflineEditsLoader { abstract public void loadEdits() throws IOException; - public abstract void setFixTxIds(); - static class OfflineEditsLoaderFactory { static OfflineEditsLoader createLoader(OfflineEditsVisitor visitor, - String inputFileName, boolean xmlInput) throws IOException { + String inputFileName, boolean xmlInput, + OfflineEditsViewer.Flags flags) throws IOException { if (xmlInput) { - return new OfflineEditsXmlLoader(visitor, new File(inputFileName)); + return new OfflineEditsXmlLoader(visitor, new File(inputFileName), flags); } else { File file = null; EditLogInputStream elis = null; @@ -51,7 +51,7 @@ static OfflineEditsLoader createLoader(OfflineEditsVisitor visitor, file = new File(inputFileName); elis = new EditLogFileInputStream(file, HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID, false); - loader = new OfflineEditsBinaryLoader(visitor, elis); + loader = new OfflineEditsBinaryLoader(visitor, elis, flags); } finally { if ((loader == null) && (elis != null)) { elis.close(); @@ -61,4 +61,4 @@ static OfflineEditsLoader createLoader(OfflineEditsVisitor visitor, } } } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java index 6fecab6c30..833f2bc05b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java @@ -17,16 +17,10 @@ */ package org.apache.hadoop.hdfs.tools.offlineEditsViewer; -import java.io.EOFException; -import java.io.File; -import java.io.IOException; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream; -import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream; import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsLoader.OfflineEditsLoaderFactory; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -37,7 +31,6 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.cli.PosixParser; -import org.xml.sax.SAXParseException; /** * This class implements an offline edits viewer, tool that @@ -78,6 +71,9 @@ private void printHelp() { "-f,--fix-txids Renumber the transaction IDs in the input,\n" + " so that there are no gaps or invalid " + " transaction IDs.\n" + + "-r,--recover When reading binary edit logs, use recovery \n" + + " mode. This will give you the chance to skip \n" + + " corrupt parts of the edit log.\n" + "-v,--verbose More verbose output, prints the input and\n" + " output filenames, for processors that write\n" + " to a file, also output to screen. On large\n" + @@ -113,6 +109,7 @@ public static Options buildOptions() { options.addOption("p", "processor", true, ""); options.addOption("v", "verbose", false, ""); options.addOption("f", "fix-txids", false, ""); + options.addOption("r", "recover", false, ""); options.addOption("h", "help", false, ""); return options; @@ -128,23 +125,20 @@ public static Options buildOptions() { * @return 0 on success; error code otherwise */ public int go(String inputFileName, String outputFileName, String processor, - boolean printToScreen, boolean fixTxIds, OfflineEditsVisitor visitor) + Flags flags, OfflineEditsVisitor visitor) { - if (printToScreen) { + if (flags.getPrintToScreen()) { System.out.println("input [" + inputFileName + "]"); System.out.println("output [" + outputFileName + "]"); } try { if (visitor == null) { visitor = OfflineEditsVisitorFactory.getEditsVisitor( - outputFileName, processor, printToScreen); + outputFileName, processor, flags.getPrintToScreen()); } boolean xmlInput = inputFileName.endsWith(".xml"); OfflineEditsLoader loader = OfflineEditsLoaderFactory. - createLoader(visitor, inputFileName, xmlInput); - if (fixTxIds) { - loader.setFixTxIds(); - } + createLoader(visitor, inputFileName, xmlInput, flags); loader.loadEdits(); } catch(Exception e) { System.err.println("Encountered exception. Exiting: " + e.getMessage()); @@ -154,6 +148,39 @@ public int go(String inputFileName, String outputFileName, String processor, return 0; } + public static class Flags { + private boolean printToScreen = false; + private boolean fixTxIds = false; + private boolean recoveryMode = false; + + public Flags() { + } + + public boolean getPrintToScreen() { + return printToScreen; + } + + public void setPrintToScreen() { + printToScreen = true; + } + + public boolean getFixTxIds() { + return fixTxIds; + } + + public void setFixTxIds() { + fixTxIds = true; + } + + public boolean getRecoveryMode() { + return recoveryMode; + } + + public void setRecoveryMode() { + recoveryMode = true; + } + } + /** * Main entry point for ToolRunner (see ToolRunner docs) * @@ -177,6 +204,7 @@ public int run(String[] argv) throws Exception { printHelp(); return -1; } + if(cmd.hasOption("h")) { // print help and exit printHelp(); return -1; @@ -187,10 +215,17 @@ public int run(String[] argv) throws Exception { if(processor == null) { processor = defaultProcessor; } - boolean printToScreen = cmd.hasOption("v"); - boolean fixTxIds = cmd.hasOption("f"); - return go(inputFileName, outputFileName, processor, - printToScreen, fixTxIds, null); + Flags flags = new Flags(); + if (cmd.hasOption("r")) { + flags.setRecoveryMode(); + } + if (cmd.hasOption("f")) { + flags.setFixTxIds(); + } + if (cmd.hasOption("v")) { + flags.setPrintToScreen(); + } + return go(inputFileName, outputFileName, processor, flags, null); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java index 009db6a477..393015ba96 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.OpInstanceCache; - +import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer; import org.apache.hadoop.hdfs.util.XMLUtils.Stanza; import org.xml.sax.Attributes; import org.xml.sax.InputSource; @@ -46,9 +46,9 @@ @InterfaceStability.Unstable class OfflineEditsXmlLoader extends DefaultHandler implements OfflineEditsLoader { - private boolean fixTxIds; - private OfflineEditsVisitor visitor; - private FileReader fileReader; + private final boolean fixTxIds; + private final OfflineEditsVisitor visitor; + private final FileReader fileReader; private ParseState state; private Stanza stanza; private Stack stanzaStack; @@ -68,9 +68,10 @@ static enum ParseState { } public OfflineEditsXmlLoader(OfflineEditsVisitor visitor, - File inputFile) throws FileNotFoundException { + File inputFile, OfflineEditsViewer.Flags flags) throws FileNotFoundException { this.visitor = visitor; this.fileReader = new FileReader(inputFile); + this.fixTxIds = flags.getFixTxIds(); } /** @@ -250,9 +251,4 @@ public void endElement (String uri, String name, String qName) { public void characters (char ch[], int start, int length) { cbuf.append(ch, start, length); } - - @Override - public void setFixTxIds() { - fixTxIds = true; - } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java index 84f55216da..a6746a249b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java @@ -22,11 +22,14 @@ import java.io.IOException; import java.io.File; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.util.Map; import java.util.HashMap; import org.junit.Test; import org.junit.Before; + +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import org.apache.commons.logging.Log; @@ -34,12 +37,12 @@ import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes; import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer; +import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer.Flags; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.server.namenode.OfflineEditsViewerHelper; public class TestOfflineEditsViewer { - private static final Log LOG = LogFactory.getLog(TestOfflineEditsViewer.class); private static final Map obsoleteOpCodes = @@ -97,8 +100,8 @@ public void testGenerated() throws IOException { String editsReparsed = cacheDir + "/editsReparsed"; // parse to XML then back to binary - runOev(edits, editsParsedXml, "xml"); - runOev(editsParsedXml, editsReparsed, "binary"); + assertEquals(0, runOev(edits, editsParsedXml, "xml", false)); + assertEquals(0, runOev(editsParsedXml, editsReparsed, "binary", false)); // judgment time assertTrue( @@ -114,6 +117,42 @@ public void testGenerated() throws IOException { LOG.info("END"); } + @Test + public void testRecoveryMode() throws IOException { + LOG.info("START - testing with generated edits"); + + nnHelper.startCluster(buildDir + "/dfs/"); + + // edits generated by nnHelper (MiniDFSCluster), should have all op codes + // binary, XML, reparsed binary + String edits = nnHelper.generateEdits(); + + // Corrupt the file by truncating the end + FileChannel editsFile = new FileOutputStream(edits, true).getChannel(); + editsFile.truncate(editsFile.size() - 5); + + String editsParsedXml = cacheDir + "/editsRecoveredParsed.xml"; + String editsReparsed = cacheDir + "/editsRecoveredReparsed"; + String editsParsedXml2 = cacheDir + "/editsRecoveredParsed2.xml"; + + // Can't read the corrupted file without recovery mode + assertEquals(-1, runOev(edits, editsParsedXml, "xml", false)); + + // parse to XML then back to binary + assertEquals(0, runOev(edits, editsParsedXml, "xml", true)); + assertEquals(0, runOev(editsParsedXml, editsReparsed, "binary", false)); + assertEquals(0, runOev(editsReparsed, editsParsedXml2, "xml", false)); + + // judgment time + assertTrue("Test round trip", + filesEqualIgnoreTrailingZeros(editsParsedXml, editsParsedXml2)); + + // removes edits so do this at the end + nnHelper.shutdownCluster(); + + LOG.info("END"); + } + @Test public void testStored() throws IOException { @@ -128,8 +167,9 @@ public void testStored() throws IOException { String editsStoredXml = cacheDir + "/editsStored.xml"; // parse to XML then back to binary - runOev(editsStored, editsStoredParsedXml, "xml"); - runOev(editsStoredParsedXml, editsStoredReparsed, "binary"); + assertEquals(0, runOev(editsStored, editsStoredParsedXml, "xml", false)); + assertEquals(0, runOev(editsStoredParsedXml, editsStoredReparsed, + "binary", false)); // judgement time assertTrue( @@ -151,14 +191,18 @@ public void testStored() throws IOException { * @param inFilename input edits filename * @param outFilename oputput edits filename */ - private void runOev(String inFilename, String outFilename, String processor) - throws IOException { + private int runOev(String inFilename, String outFilename, String processor, + boolean recovery) throws IOException { LOG.info("Running oev [" + inFilename + "] [" + outFilename + "]"); OfflineEditsViewer oev = new OfflineEditsViewer(); - if (oev.go(inFilename, outFilename, processor, true, false, null) != 0) - throw new RuntimeException("oev failed"); + Flags flags = new Flags(); + flags.setPrintToScreen(); + if (recovery) { + flags.setRecoveryMode(); + } + return oev.go(inFilename, outFilename, processor, flags, null); } /** @@ -172,7 +216,7 @@ private boolean hasAllOpCodes(String inFilename) throws IOException { FileOutputStream fout = new FileOutputStream(outFilename); StatisticsEditsVisitor visitor = new StatisticsEditsVisitor(fout); OfflineEditsViewer oev = new OfflineEditsViewer(); - if (oev.go(inFilename, outFilename, "stats", false, false, visitor) != 0) + if (oev.go(inFilename, outFilename, "stats", new Flags(), visitor) != 0) return false; LOG.info("Statistics for " + inFilename + "\n" + visitor.getStatisticsString()); From 1585084ffb746f22cd84855ebfad67a80c521965 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Wed, 13 Jun 2012 05:04:20 +0000 Subject: [PATCH 43/91] HDFS-3478. Test quotas with Long.Max_Value. Contributed by Sujay Rau git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349634 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../org/apache/hadoop/hdfs/TestQuota.java | 59 +++++++++++++++++-- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 450ba4a0ef..20ecf86753 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -93,6 +93,8 @@ Trunk (unreleased changes) HDFS-3049. During the normal NN startup process, fall back on a different edit log if we see one that is corrupt (Colin Patrick McCabe via todd) + HDFS-3478. Test quotas with Long.Max_Value. (Sujay Rau via eli) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java index a747a333fb..19818a6633 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java @@ -17,9 +17,7 @@ */ package org.apache.hadoop.hdfs; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import java.io.OutputStream; import java.security.PrivilegedExceptionAction; @@ -773,7 +771,60 @@ private static void checkContentSummary(final ContentSummary expected, final ContentSummary computed) { assertEquals(expected.toString(), computed.toString()); } - + + /** + * Test limit cases for setting space quotas. + */ + @Test + public void testMaxSpaceQuotas() throws Exception { + final Configuration conf = new HdfsConfiguration(); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + final FileSystem fs = cluster.getFileSystem(); + assertTrue("Not a HDFS: "+fs.getUri(), + fs instanceof DistributedFileSystem); + final DistributedFileSystem dfs = (DistributedFileSystem)fs; + + // create test directory + final Path testFolder = new Path("/testFolder"); + assertTrue(dfs.mkdirs(testFolder)); + + // setting namespace quota to Long.MAX_VALUE - 1 should work + dfs.setQuota(testFolder, Long.MAX_VALUE - 1, 10); + ContentSummary c = dfs.getContentSummary(testFolder); + assertTrue("Quota not set properly", c.getQuota() == Long.MAX_VALUE - 1); + + // setting diskspace quota to Long.MAX_VALUE - 1 should work + dfs.setQuota(testFolder, 10, Long.MAX_VALUE - 1); + c = dfs.getContentSummary(testFolder); + assertTrue("Quota not set properly", c.getSpaceQuota() == Long.MAX_VALUE - 1); + + // setting namespace quota to Long.MAX_VALUE should not work + no error + dfs.setQuota(testFolder, Long.MAX_VALUE, 10); + c = dfs.getContentSummary(testFolder); + assertTrue("Quota should not have changed", c.getQuota() == 10); + + // setting diskspace quota to Long.MAX_VALUE should not work + no error + dfs.setQuota(testFolder, 10, Long.MAX_VALUE); + c = dfs.getContentSummary(testFolder); + assertTrue("Quota should not have changed", c.getSpaceQuota() == 10); + + // setting namespace quota to Long.MAX_VALUE + 1 should not work + error + try { + dfs.setQuota(testFolder, Long.MAX_VALUE + 1, 10); + fail("Exception not thrown"); + } catch (IllegalArgumentException e) { + // Expected + } + + // setting diskspace quota to Long.MAX_VALUE + 1 should not work + error + try { + dfs.setQuota(testFolder, 10, Long.MAX_VALUE + 1); + fail("Exception not thrown"); + } catch (IllegalArgumentException e) { + // Expected + } + } + /** * Violate a space quota using files of size < 1 block. Test that block * allocation conservatively assumes that for quota checking the entire From 48ebef5141e405f81684add7698cfbcea29df6e4 Mon Sep 17 00:00:00 2001 From: Aaron Twining Myers Date: Wed, 13 Jun 2012 05:29:57 +0000 Subject: [PATCH 44/91] HDFS-766. Error message not clear for set space quota out of boundary values. Contributed by Jon Zuanich. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349639 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java | 7 ++++++- .../hadoop-hdfs/src/test/resources/testHDFSConf.xml | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 20ecf86753..a1420bd3be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -336,6 +336,9 @@ Branch-2 ( Unreleased changes ) HDFS-3531. EditLogFileOutputStream#preallocate should check for incomplete writes. (Colin Patrick McCabe via eli) + HDFS-766. Error message not clear for set space quota out of boundary + values. (Jon Zuanich via atm) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index c7c206fcb7..211220803e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -239,7 +239,12 @@ private static class SetSpaceQuotaCommand extends DFSAdminCommand { CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE); List parameters = c.parse(args, pos); String str = parameters.remove(0).trim(); - quota = StringUtils.TraditionalBinaryPrefix.string2long(str); + try { + quota = StringUtils.TraditionalBinaryPrefix.string2long(str); + } catch (NumberFormatException nfe) { + throw new IllegalArgumentException("\"" + str + "\" is not a valid value for a quota."); + } + this.args = parameters.toArray(new String[parameters.size()]); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index 918c004516..1f6a6265ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -15470,7 +15470,7 @@ SubstringComparator - For input string: "a5" + setSpaceQuota: "a5" is not a valid value for a quota. From f737736527b47b44d3e9b4a73d6c00d48082d021 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Wed, 13 Jun 2012 05:37:27 +0000 Subject: [PATCH 45/91] HADOOP-8433. Don't set HADOOP_LOG_DIR in hadoop-env.sh. Contributed by Brahma Reddy Battula git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349641 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop-common/src/main/conf/hadoop-env.sh | 2 +- .../src/main/packages/templates/conf/hadoop-env.sh | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index aa28d7dea0..aa766b674e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -253,6 +253,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8507. Avoid OOM while deserializing DelegationTokenIdentifer. (Colin Patrick McCabe via eli) + HADOOP-8433. Don't set HADOOP_LOG_DIR in hadoop-env.sh. + (Brahma Reddy Battula via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh index 33abeca1ac..72e8c63f66 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh @@ -61,7 +61,7 @@ export HADOOP_CLIENT_OPTS="-Xmx128m $HADOOP_CLIENT_OPTS" export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER} # Where log files are stored. $HADOOP_HOME/logs by default. -export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER +#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER # Where log files are stored in the secure data environment. export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER} diff --git a/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-env.sh index d8c731ec5f..77693fb1ec 100644 --- a/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-env.sh +++ b/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-env.sh @@ -65,7 +65,7 @@ export HADOOP_CLIENT_OPTS="-Xmx128m $HADOOP_CLIENT_OPTS" export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER} # Where log files are stored. $HADOOP_HOME/logs by default. -export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER +#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER # Where log files are stored in the secure data environment. export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER} From 9d91a40e245dfcceaf7f920a7c06f9e73c84f16d Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Wed, 13 Jun 2012 05:53:24 +0000 Subject: [PATCH 46/91] HDFS-3480. Multiple SLF4J binding warning. Contributed by Vinay git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1349643 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ hadoop-hdfs-project/hadoop-hdfs/pom.xml | 21 ++++++--------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a1420bd3be..e17614283d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -339,6 +339,8 @@ Branch-2 ( Unreleased changes ) HDFS-766. Error message not clear for set space quota out of boundary values. (Jon Zuanich via atm) + HDFS-3480. Multiple SLF4J binding warning. (Vinay via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index c775c51e3f..41bcf31baf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -107,21 +107,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.zookeeper zookeeper 3.4.2 - - - - junit - junit - - - com.sun.jdmk - jmxtools - - - com.sun.jmx - jmxri - - + provided + + + org.slf4j + slf4j-log4j12 + provided org.apache.zookeeper From 836f2c4c996084daa8b0a9dff1ee5e758f75181f Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 13 Jun 2012 20:47:51 +0000 Subject: [PATCH 47/91] HADOOP-8509. JarFinder duplicate entry: META-INF/MANIFEST.MF exception (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1350008 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../org/apache/hadoop/util/JarFinder.java | 53 +++++++---- .../org/apache/hadoop/util/TestJarFinder.java | 88 ++++++++++++++++++- 3 files changed, 126 insertions(+), 17 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index aa766b674e..08f240c665 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -256,6 +256,8 @@ Branch-2 ( Unreleased changes ) HADOOP-8433. Don't set HADOOP_LOG_DIR in hadoop-env.sh. (Brahma Reddy Battula via eli) + HADOOP-8509. JarFinder duplicate entry: META-INF/MANIFEST.MF exception (tucu) + BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/JarFinder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/JarFinder.java index 37a561eaf7..717838251c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/JarFinder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/JarFinder.java @@ -15,15 +15,18 @@ import com.google.common.base.Preconditions; +import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.net.URL; import java.net.URLDecoder; import java.text.MessageFormat; import java.util.Enumeration; +import java.util.jar.JarFile; import java.util.jar.JarOutputStream; import java.util.jar.Manifest; import java.util.zip.ZipEntry; @@ -37,10 +40,37 @@ */ public class JarFinder { - private static void zipDir(File dir, String relativePath, ZipOutputStream zos) + private static void copyToZipStream(InputStream is, ZipEntry entry, + ZipOutputStream zos) throws IOException { + zos.putNextEntry(entry); + byte[] arr = new byte[4096]; + int read = is.read(arr); + while (read > -1) { + zos.write(arr, 0, read); + read = is.read(arr); + } + is.close(); + zos.closeEntry(); + } + + public static void jarDir(File dir, String relativePath, ZipOutputStream zos) throws IOException { Preconditions.checkNotNull(relativePath, "relativePath"); Preconditions.checkNotNull(zos, "zos"); + + // by JAR spec, if there is a manifest, it must be the first entry in the + // ZIP. + File manifestFile = new File(dir, JarFile.MANIFEST_NAME); + ZipEntry manifestEntry = new ZipEntry(JarFile.MANIFEST_NAME); + if (!manifestFile.exists()) { + zos.putNextEntry(manifestEntry); + new Manifest().write(new BufferedOutputStream(zos)); + zos.closeEntry(); + } else { + InputStream is = new FileInputStream(manifestFile); + copyToZipStream(is, manifestEntry, zos); + } + zos.closeEntry(); zipDir(dir, relativePath, zos, true); zos.close(); } @@ -62,17 +92,12 @@ private static void zipDir(File dir, String relativePath, ZipOutputStream zos, zipDir(file, relativePath + f.getName() + "/", zos, false); } else { - ZipEntry anEntry = new ZipEntry(relativePath + f.getName()); - zos.putNextEntry(anEntry); - InputStream is = new FileInputStream(f); - byte[] arr = new byte[4096]; - int read = is.read(arr); - while (read > -1) { - zos.write(arr, 0, read); - read = is.read(arr); + String path = relativePath + f.getName(); + if (!path.equals(JarFile.MANIFEST_NAME)) { + ZipEntry anEntry = new ZipEntry(path); + InputStream is = new FileInputStream(f); + copyToZipStream(is, anEntry, zos); } - is.close(); - zos.closeEntry(); } } } @@ -88,9 +113,8 @@ private static void createJar(File dir, File jarFile) throws IOException { jarDir)); } } - JarOutputStream zos = new JarOutputStream(new FileOutputStream(jarFile), - new Manifest()); - zipDir(dir, "", zos); + JarOutputStream zos = new JarOutputStream(new FileOutputStream(jarFile)); + jarDir(dir, "", zos); } /** @@ -142,5 +166,4 @@ else if ("file".equals(url.getProtocol())) { } return null; } - } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJarFinder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJarFinder.java index a311a9f712..4997b7a824 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJarFinder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJarFinder.java @@ -22,21 +22,105 @@ import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.Writer; +import java.text.MessageFormat; +import java.util.Properties; +import java.util.jar.JarInputStream; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; public class TestJarFinder { @Test - public void testAppend() throws Exception { + public void testJar() throws Exception { //picking a class that is for sure in a JAR in the classpath String jar = JarFinder.getJar(LogFactory.class); Assert.assertTrue(new File(jar).exists()); + } + + private static void delete(File file) throws IOException { + if (file.getAbsolutePath().length() < 5) { + throw new IllegalArgumentException( + MessageFormat.format("Path [{0}] is too short, not deleting", + file.getAbsolutePath())); + } + if (file.exists()) { + if (file.isDirectory()) { + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + delete(child); + } + } + } + if (!file.delete()) { + throw new RuntimeException( + MessageFormat.format("Could not delete path [{0}]", + file.getAbsolutePath())); + } + } + } + @Test + public void testExpandedClasspath() throws Exception { //picking a class that is for sure in a directory in the classpath //in this case the JAR is created on the fly - jar = JarFinder.getJar(TestJarFinder.class); + String jar = JarFinder.getJar(TestJarFinder.class); Assert.assertTrue(new File(jar).exists()); } + @Test + public void testExistingManifest() throws Exception { + File dir = new File(System.getProperty("test.build.dir", "target/test-dir"), + TestJarFinder.class.getName() + "-testExistingManifest"); + delete(dir); + dir.mkdirs(); + + File metaInfDir = new File(dir, "META-INF"); + metaInfDir.mkdirs(); + File manifestFile = new File(metaInfDir, "MANIFEST.MF"); + Manifest manifest = new Manifest(); + OutputStream os = new FileOutputStream(manifestFile); + manifest.write(os); + os.close(); + + File propsFile = new File(dir, "props.properties"); + Writer writer = new FileWriter(propsFile); + new Properties().store(writer, ""); + writer.close(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JarOutputStream zos = new JarOutputStream(baos); + JarFinder.jarDir(dir, "", zos); + JarInputStream jis = + new JarInputStream(new ByteArrayInputStream(baos.toByteArray())); + Assert.assertNotNull(jis.getManifest()); + jis.close(); + } + + @Test + public void testNoManifest() throws Exception { + File dir = new File(System.getProperty("test.build.dir", "target/test-dir"), + TestJarFinder.class.getName() + "-testNoManifest"); + delete(dir); + dir.mkdirs(); + File propsFile = new File(dir, "props.properties"); + Writer writer = new FileWriter(propsFile); + new Properties().store(writer, ""); + writer.close(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JarOutputStream zos = new JarOutputStream(baos); + JarFinder.jarDir(dir, "", zos); + JarInputStream jis = + new JarInputStream(new ByteArrayInputStream(baos.toByteArray())); + Assert.assertNotNull(jis.getManifest()); + jis.close(); + } } From b2a8cf3759646524426ca9f9c11974f14986a0a0 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Thu, 14 Jun 2012 20:17:54 +0000 Subject: [PATCH 48/91] HDFS-3524. Update TestFileLengthOnClusterRestart for HDFS-3522. Contributed by Brandon Li git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1350384 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../apache/hadoop/hdfs/TestFileLengthOnClusterRestart.java | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e17614283d..c1c434d761 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -341,6 +341,9 @@ Branch-2 ( Unreleased changes ) HDFS-3480. Multiple SLF4J binding warning. (Vinay via eli) + HDFS-3524. Update TestFileLengthOnClusterRestart for HDFS-3522. (Brandon + Li via szetszwo) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileLengthOnClusterRestart.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileLengthOnClusterRestart.java index 5f9ad3259d..f63ba9a53a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileLengthOnClusterRestart.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileLengthOnClusterRestart.java @@ -65,8 +65,8 @@ public void testFileLengthWithHSyncAndClusterRestartWithOutDNsRegister() in = (HdfsDataInputStream) dfs.open(path); Assert.fail("Expected IOException"); } catch (IOException e) { - Assert.assertEquals("Could not obtain the last block locations.", e - .getLocalizedMessage()); + Assert.assertTrue(e.getLocalizedMessage().indexOf( + "Name node is in safe mode") >= 0); } } finally { if (null != in) { From 015c5cc448d7bac2aca28ee2e199963bbbba768b Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Fri, 15 Jun 2012 18:35:58 +0000 Subject: [PATCH 49/91] Updated CHANGES.txt for HDFS-3408(BKJM : Namenode format fails, if there is no BK root) And HDFS-3389(Document the BKJM usage in Namenode HA) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1350733 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index c1c434d761..461850c54f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -332,6 +332,10 @@ Branch-2 ( Unreleased changes ) HDFS-3522. If a namenode is in safemode, it should throw SafeModeException when getBlockLocations has zero locations. (Brandon Li via szetszwo) + + HDFS-3408. BKJM : Namenode format fails, if there is no BK root. (Rakesh R via umamahesh) + + HDFS-3389. Document the BKJM usage in Namenode HA. (umamahesh and Ivan Kelly via umamahesh) HDFS-3531. EditLogFileOutputStream#preallocate should check for incomplete writes. (Colin Patrick McCabe via eli) From baa0516120aa3d8330573047f75e16cfeaeb9dd3 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Fri, 15 Jun 2012 23:00:36 +0000 Subject: [PATCH 50/91] HDFS-3518. Add a utility method HdfsUtils.isHealthy(uri) for checking if the given HDFS is healthy. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1350825 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 7 +- .../hadoop/hdfs/DistributedFileSystem.java | 2 +- .../apache/hadoop/hdfs/client/HdfsUtils.java | 87 +++++++++++++++++++ .../hadoop/hdfs/TestDFSClientRetries.java | 16 +++- 4 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsUtils.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 461850c54f..fc632047c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -13,8 +13,6 @@ Trunk (unreleased changes) HDFS-3125. Add JournalService to enable Journal Daemon. (suresh) - HDFS-744. Support hsync in HDFS. (Lars Hofhansl via szetszwo) - IMPROVEMENTS HDFS-1620. Rename HdfsConstants -> HdfsServerConstants, FSConstants -> @@ -162,9 +160,14 @@ Branch-2 ( Unreleased changes ) NEW FEATURES + HDFS-744. Support hsync in HDFS. (Lars Hofhansl via szetszwo) + HDFS-3042. Automatic failover support for NameNode HA (todd) (see dedicated section below for breakdown of subtasks) + HDFS-3518. Add a utility method HdfsUtils.isHealthy(uri) for checking if + the given HDFS is healthy. (szetszwo) + IMPROVEMENTS HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 1de353bdb8..8c0ed10d2b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -96,7 +96,7 @@ public DistributedFileSystem() { */ @Override public String getScheme() { - return "hdfs"; + return HdfsConstants.HDFS_URI_SCHEME; } @Deprecated diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsUtils.java new file mode 100644 index 0000000000..9133cf3181 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsUtils.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.client; + +import java.io.IOException; +import java.net.URI; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; +import org.apache.hadoop.io.IOUtils; + +/** + * The public utility API for HDFS. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class HdfsUtils { + private static final Log LOG = LogFactory.getLog(HdfsUtils.class); + + /** + * Is the HDFS healthy? + * HDFS is considered as healthy if it is up and not in safemode. + * + * @param uri the HDFS URI. Note that the URI path is ignored. + * @return true if HDFS is healthy; false, otherwise. + */ + public static boolean isHealthy(URI uri) { + //check scheme + final String scheme = uri.getScheme(); + if (!HdfsConstants.HDFS_URI_SCHEME.equalsIgnoreCase(scheme)) { + throw new IllegalArgumentException("The scheme is not " + + HdfsConstants.HDFS_URI_SCHEME + ", uri=" + uri); + } + + final Configuration conf = new Configuration(); + //disable FileSystem cache + conf.setBoolean(String.format("fs.%s.impl.disable.cache", scheme), true); + //disable client retry for rpc connection and rpc calls + conf.setBoolean(DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, false); + conf.setInt( + CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); + + DistributedFileSystem fs = null; + try { + fs = (DistributedFileSystem)FileSystem.get(uri, conf); + final boolean safemode = fs.setSafeMode(SafeModeAction.SAFEMODE_GET); + if (LOG.isDebugEnabled()) { + LOG.debug("Is namenode in safemode? " + safemode + "; uri=" + uri); + } + + fs.close(); + fs = null; + return !safemode; + } catch(IOException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Got an exception for uri=" + uri, e); + } + return false; + } finally { + IOUtils.cleanup(LOG, fs); + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java index 4d8244dde7..44e4f9fd23 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java @@ -31,6 +31,7 @@ import java.io.OutputStream; import java.net.InetSocketAddress; import java.net.SocketTimeoutException; +import java.net.URI; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; @@ -51,6 +52,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.client.HdfsUtils; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeID; @@ -823,9 +825,11 @@ public void testNamenodeRestart() throws Exception { .build(); try { cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final URI uri = dfs.getUri(); + assertTrue(HdfsUtils.isHealthy(uri)); //create a file - final DistributedFileSystem dfs = cluster.getFileSystem(); final long length = 1L << 20; final Path file1 = new Path(dir, "foo"); DFSTestUtil.createFile(dfs, file1, length, numDatanodes, 20120406L); @@ -835,7 +839,9 @@ public void testNamenodeRestart() throws Exception { assertEquals(length, s1.getLen()); //shutdown namenode + assertTrue(HdfsUtils.isHealthy(uri)); cluster.shutdownNameNode(0); + assertFalse(HdfsUtils.isHealthy(uri)); //namenode is down, create another file in a thread final Path file3 = new Path(dir, "file"); @@ -860,8 +866,10 @@ public void run() { try { //sleep, restart, and then wait active TimeUnit.SECONDS.sleep(30); + assertFalse(HdfsUtils.isHealthy(uri)); cluster.restartNameNode(0, false); cluster.waitActive(); + assertTrue(HdfsUtils.isHealthy(uri)); } catch (Exception e) { exceptions.add(e); } @@ -877,7 +885,9 @@ public void run() { assertEquals(dfs.getFileChecksum(file1), dfs.getFileChecksum(file3)); //enter safe mode + assertTrue(HdfsUtils.isHealthy(uri)); dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + assertFalse(HdfsUtils.isHealthy(uri)); //leave safe mode in a new thread new Thread(new Runnable() { @@ -886,7 +896,9 @@ public void run() { try { //sleep and then leave safe mode TimeUnit.SECONDS.sleep(30); + assertFalse(HdfsUtils.isHealthy(uri)); dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + assertTrue(HdfsUtils.isHealthy(uri)); } catch (Exception e) { exceptions.add(e); } @@ -898,6 +910,8 @@ public void run() { DFSTestUtil.createFile(dfs, file2, length, numDatanodes, 20120406L); assertEquals(dfs.getFileChecksum(file1), dfs.getFileChecksum(file2)); + assertTrue(HdfsUtils.isHealthy(uri)); + //make sure it won't retry on exceptions like FileNotFoundException final Path nonExisting = new Path(dir, "nonExisting"); LOG.info("setPermission: " + nonExisting); From 7debb334f8ce96ca809cd776ddeca240218d8aee Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Sun, 17 Jun 2012 21:12:25 +0000 Subject: [PATCH 51/91] HADOOP-8468. Add NetworkTopologyWithNodeGroup, a 4-layer implementation of NetworkTopology. Contributed by Junping Du git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351163 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../net/NetworkTopologyWithNodeGroup.java | 398 ++++++++++++++++++ .../src/main/resources/core-default.xml | 12 +- .../net/TestNetworkTopologyWithNodeGroup.java | 165 ++++++++ 4 files changed, 575 insertions(+), 3 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 08f240c665..013d53137f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -12,6 +12,9 @@ Trunk (unreleased changes) HADOOP-8469. Make NetworkTopology class pluggable. (Junping Du via szetszwo) + HADOOP-8468. Add NetworkTopologyWithNodeGroup, a 4-layer implementation + of NetworkTopology. (Junping Du via szetszwo) + IMPROVEMENTS HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java new file mode 100644 index 0000000000..6066cd2a61 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java @@ -0,0 +1,398 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.net; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * The class extends NetworkTopology to represents a cluster of computer with + * a 4-layers hierarchical network topology. + * In this network topology, leaves represent data nodes (computers) and inner + * nodes represent switches/routers that manage traffic in/out of data centers, + * racks or physical host (with virtual switch). + * + * @see NetworkTopology + */ +@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) +@InterfaceStability.Unstable +public class NetworkTopologyWithNodeGroup extends NetworkTopology { + + public final static String DEFAULT_NODEGROUP = "/default-nodegroup"; + + public NetworkTopologyWithNodeGroup() { + clusterMap = new InnerNodeWithNodeGroup(InnerNode.ROOT); + } + + @Override + protected Node getNodeForNetworkLocation(Node node) { + // if node only with default rack info, here we need to add default + // nodegroup info + if (NetworkTopology.DEFAULT_RACK.equals(node.getNetworkLocation())) { + node.setNetworkLocation(node.getNetworkLocation() + + DEFAULT_NODEGROUP); + } + Node nodeGroup = getNode(node.getNetworkLocation()); + if (nodeGroup == null) { + nodeGroup = new InnerNode(node.getNetworkLocation()); + } + return getNode(nodeGroup.getNetworkLocation()); + } + + @Override + public String getRack(String loc) { + netlock.readLock().lock(); + try { + loc = InnerNode.normalize(loc); + Node locNode = getNode(loc); + if (locNode instanceof InnerNodeWithNodeGroup) { + InnerNodeWithNodeGroup node = (InnerNodeWithNodeGroup) locNode; + if (node.isRack()) { + return loc; + } else if (node.isNodeGroup()) { + return node.getNetworkLocation(); + } else { + // may be a data center + return null; + } + } else { + // not in cluster map, don't handle it + return loc; + } + } finally { + netlock.readLock().unlock(); + } + } + + /** + * Given a string representation of a node group for a specific network + * location + * + * @param loc + * a path-like string representation of a network location + * @return a node group string + */ + public String getNodeGroup(String loc) { + netlock.readLock().lock(); + try { + loc = InnerNode.normalize(loc); + Node locNode = getNode(loc); + if (locNode instanceof InnerNodeWithNodeGroup) { + InnerNodeWithNodeGroup node = (InnerNodeWithNodeGroup) locNode; + if (node.isNodeGroup()) { + return loc; + } else if (node.isRack()) { + // not sure the node group for a rack + return null; + } else { + // may be a leaf node + return getNodeGroup(node.getNetworkLocation()); + } + } else { + // not in cluster map, don't handle it + return loc; + } + } finally { + netlock.readLock().unlock(); + } + } + + @Override + public boolean isOnSameRack( Node node1, Node node2) { + if (node1 == null || node2 == null || + node1.getParent() == null || node2.getParent() == null) { + return false; + } + + netlock.readLock().lock(); + try { + return isSameParents(node1.getParent(), node2.getParent()); + } finally { + netlock.readLock().unlock(); + } + } + + /** + * Check if two nodes are on the same node group (hypervisor) The + * assumption here is: each nodes are leaf nodes. + * + * @param node1 + * one node (can be null) + * @param node2 + * another node (can be null) + * @return true if node1 and node2 are on the same node group; false + * otherwise + * @exception IllegalArgumentException + * when either node1 or node2 is null, or node1 or node2 do + * not belong to the cluster + */ + @Override + public boolean isOnSameNodeGroup(Node node1, Node node2) { + if (node1 == null || node2 == null) { + return false; + } + netlock.readLock().lock(); + try { + return isSameParents(node1, node2); + } finally { + netlock.readLock().unlock(); + } + } + + /** + * Check if network topology is aware of NodeGroup + */ + @Override + public boolean isNodeGroupAware() { + return true; + } + + /** Add a leaf node + * Update node counter & rack counter if necessary + * @param node node to be added; can be null + * @exception IllegalArgumentException if add a node to a leave + * or node to be added is not a leaf + */ + @Override + public void add(Node node) { + if (node==null) return; + if( node instanceof InnerNode ) { + throw new IllegalArgumentException( + "Not allow to add an inner node: "+NodeBase.getPath(node)); + } + netlock.writeLock().lock(); + try { + Node rack = null; + + // if node only with default rack info, here we need to add default + // nodegroup info + if (NetworkTopology.DEFAULT_RACK.equals(node.getNetworkLocation())) { + node.setNetworkLocation(node.getNetworkLocation() + + NetworkTopologyWithNodeGroup.DEFAULT_NODEGROUP); + } + Node nodeGroup = getNode(node.getNetworkLocation()); + if (nodeGroup == null) { + nodeGroup = new InnerNodeWithNodeGroup(node.getNetworkLocation()); + } + rack = getNode(nodeGroup.getNetworkLocation()); + + if (rack != null && !(rack instanceof InnerNode)) { + throw new IllegalArgumentException("Unexpected data node " + + node.toString() + + " at an illegal network location"); + } + if (clusterMap.add(node)) { + LOG.info("Adding a new node: " + NodeBase.getPath(node)); + if (rack == null) { + // We only track rack number here + numOfRacks++; + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("NetworkTopology became:\n" + this.toString()); + } + } finally { + netlock.writeLock().unlock(); + } + } + + /** Remove a node + * Update node counter and rack counter if necessary + * @param node node to be removed; can be null + */ + @Override + public void remove(Node node) { + if (node==null) return; + if( node instanceof InnerNode ) { + throw new IllegalArgumentException( + "Not allow to remove an inner node: "+NodeBase.getPath(node)); + } + LOG.info("Removing a node: "+NodeBase.getPath(node)); + netlock.writeLock().lock(); + try { + if (clusterMap.remove(node)) { + Node nodeGroup = getNode(node.getNetworkLocation()); + if (nodeGroup == null) { + nodeGroup = new InnerNode(node.getNetworkLocation()); + } + InnerNode rack = (InnerNode)getNode(nodeGroup.getNetworkLocation()); + if (rack == null) { + numOfRacks--; + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("NetworkTopology became:\n" + this.toString()); + } + } finally { + netlock.writeLock().unlock(); + } + } + + /** Sort nodes array by their distances to reader + * It linearly scans the array, if a local node is found, swap it with + * the first element of the array. + * If a local node group node is found, swap it with the first element + * following the local node. + * If a local rack node is found, swap it with the first element following + * the local node group node. + * If neither local node, node group node or local rack node is found, put a + * random replica location at position 0. + * It leaves the rest nodes untouched. + * @param reader the node that wishes to read a block from one of the nodes + * @param nodes the list of nodes containing data for the reader + */ + @Override + public void pseudoSortByDistance( Node reader, Node[] nodes ) { + + if (reader != null && !this.contains(reader)) { + // if reader is not a datanode (not in NetworkTopology tree), we will + // replace this reader with a sibling leaf node in tree. + Node nodeGroup = getNode(reader.getNetworkLocation()); + if (nodeGroup != null && nodeGroup instanceof InnerNode) { + InnerNode parentNode = (InnerNode) nodeGroup; + // replace reader with the first children of its parent in tree + reader = parentNode.getLeaf(0, null); + } else { + return; + } + } + int tempIndex = 0; + int localRackNode = -1; + int localNodeGroupNode = -1; + if (reader != null) { + //scan the array to find the local node & local rack node + for (int i = 0; i < nodes.length; i++) { + if (tempIndex == 0 && reader == nodes[i]) { //local node + //swap the local node and the node at position 0 + if (i != 0) { + swap(nodes, tempIndex, i); + } + tempIndex=1; + + if (localRackNode != -1 && (localNodeGroupNode !=-1)) { + if (localRackNode == 0) { + localRackNode = i; + } + if (localNodeGroupNode == 0) { + localNodeGroupNode = i; + } + break; + } + } else if (localNodeGroupNode == -1 && isOnSameNodeGroup(reader, + nodes[i])) { + //local node group + localNodeGroupNode = i; + // node local and rack local are already found + if(tempIndex != 0 && localRackNode != -1) break; + } else if (localRackNode == -1 && isOnSameRack(reader, nodes[i])) { + localRackNode = i; + if (tempIndex != 0 && localNodeGroupNode != -1) break; + } + } + + // swap the local nodegroup node and the node at position tempIndex + if(localNodeGroupNode != -1 && localNodeGroupNode != tempIndex) { + swap(nodes, tempIndex, localNodeGroupNode); + if (localRackNode == tempIndex) { + localRackNode = localNodeGroupNode; + } + tempIndex++; + } + + // swap the local rack node and the node at position tempIndex + if(localRackNode != -1 && localRackNode != tempIndex) { + swap(nodes, tempIndex, localRackNode); + tempIndex++; + } + } + + // put a random node at position 0 if there is not a local/local-nodegroup/ + // local-rack node + if (tempIndex == 0 && localNodeGroupNode == -1 && localRackNode == -1 + && nodes.length != 0) { + swap(nodes, 0, r.nextInt(nodes.length)); + } + } + + /** InnerNodeWithNodeGroup represents a switch/router of a data center, rack + * or physical host. Different from a leaf node, it has non-null children. + */ + static class InnerNodeWithNodeGroup extends InnerNode { + public InnerNodeWithNodeGroup(String name, String location, + InnerNode parent, int level) { + super(name, location, parent, level); + } + + public InnerNodeWithNodeGroup(String name, String location) { + super(name, location); + } + + public InnerNodeWithNodeGroup(String path) { + super(path); + } + + @Override + boolean isRack() { + // it is node group + if (getChildren().isEmpty()) { + return false; + } + + Node firstChild = children.get(0); + + if (firstChild instanceof InnerNode) { + Node firstGrandChild = (((InnerNode) firstChild).children).get(0); + if (firstGrandChild instanceof InnerNode) { + // it is datacenter + return false; + } else { + return true; + } + } + return false; + } + + /** + * Judge if this node represents a node group + * + * @return true if it has no child or its children are not InnerNodes + */ + boolean isNodeGroup() { + if (children.isEmpty()) { + return true; + } + Node firstChild = children.get(0); + if (firstChild instanceof InnerNode) { + // it is rack or datacenter + return false; + } + return true; + } + + @Override + protected InnerNode createParentNode(String parentName) { + return new InnerNodeWithNodeGroup(parentName, getPath(this), this, + this.getLevel() + 1); + } + + @Override + protected boolean areChildrenLeaves() { + return isNodeGroup(); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 1e72e362e7..c968ff2be6 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -599,10 +599,9 @@ - - + - net.topology.node.switch.mapping.impl + net.topology.node.switch.mapping.impl org.apache.hadoop.net.ScriptBasedMapping The default implementation of the DNSToSwitchMapping. It invokes a script specified in net.topology.script.file.name to resolve @@ -611,6 +610,13 @@ + + net.topology.impl + org.apache.hadoop.net.NetworkTopology + The default implementation of NetworkTopology which is classic three layer one. + + + net.topology.script.file.name diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java new file mode 100644 index 0000000000..7dbd33ab95 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.net; + +import java.util.HashMap; +import java.util.Map; + +import junit.framework.TestCase; + +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; + +public class TestNetworkTopologyWithNodeGroup extends TestCase { + private final static NetworkTopologyWithNodeGroup cluster = new + NetworkTopologyWithNodeGroup(); + + private final static DatanodeDescriptor dataNodes[] = new DatanodeDescriptor[] { + DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/d1/r1/s1"), + DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/d1/r1/s1"), + DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/d1/r1/s2"), + DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/d1/r2/s3"), + DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/d1/r2/s3"), + DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/d1/r2/s4"), + DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/d2/r3/s5"), + DFSTestUtil.getDatanodeDescriptor("8.8.8.8", "/d2/r3/s6") + }; + + private final static NodeBase computeNode = new NodeBase("/d1/r1/s1/h9"); + + static { + for(int i=0; i pickNodesAtRandom(int numNodes, + String excludedScope) { + Map frequency = new HashMap(); + for (DatanodeDescriptor dnd : dataNodes) { + frequency.put(dnd, 0); + } + + for (int j = 0; j < numNodes; j++) { + Node random = cluster.chooseRandom(excludedScope); + frequency.put(random, frequency.get(random) + 1); + } + return frequency; + } + + /** + * This test checks that chooseRandom works for an excluded node. + */ + public void testChooseRandomExcludedNode() { + String scope = "~" + NodeBase.getPath(dataNodes[0]); + Map frequency = pickNodesAtRandom(100, scope); + + for (Node key : dataNodes) { + // all nodes except the first should be more than zero + assertTrue(frequency.get(key) > 0 || key == dataNodes[0]); + } + } + +} From 7b1bb2a5f4290699f40c6aab223c09185893ad9e Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Mon, 18 Jun 2012 14:42:25 +0000 Subject: [PATCH 52/91] HADOOP-8495. Update Netty to avoid leaking file descriptors during shuffle (Jason Lowe via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351363 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ hadoop-project/pom.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 013d53137f..885628871b 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -722,6 +722,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8373. Port RPC.getServerAddress to 0.23 (Daryn Sharp via bobby) + HADOOP-8495. Update Netty to avoid leaking file descriptors during shuffle + (Jason Lowe via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 1aa3e40fee..5378c61911 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -380,7 +380,7 @@ org.jboss.netty netty - 3.2.3.Final + 3.2.4.Final From 8f9d2cde8eb17dfa3da2c7cc72e1cbcabdf36e23 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 18 Jun 2012 16:28:52 +0000 Subject: [PATCH 53/91] HADOOP-8512. AuthenticatedURL should reset the Token when the server returns other than OK on authentication (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351408 13f79535-47bb-0310-9956-ffa450edef68 --- .../security/authentication/client/AuthenticatedURL.java | 1 + .../authentication/client/TestAuthenticatedURL.java | 6 +++++- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java index 5a446609c2..3c59d40b3a 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java @@ -266,6 +266,7 @@ public static void extractToken(HttpURLConnection conn, Token token) throws IOEx } } } else { + token.set(null); throw new AuthenticationException("Authentication failed, status: " + conn.getResponseCode() + ", message: " + conn.getResponseMessage()); } diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java index 525af62606..213818793f 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java @@ -13,6 +13,7 @@ */ package org.apache.hadoop.security.authentication.client; +import junit.framework.Assert; import junit.framework.TestCase; import org.mockito.Mockito; @@ -100,11 +101,14 @@ public void testExtractTokenFail() throws Exception { headers.put("Set-Cookie", cookies); Mockito.when(conn.getHeaderFields()).thenReturn(headers); + AuthenticatedURL.Token token = new AuthenticatedURL.Token(); + token.set("bar"); try { - AuthenticatedURL.extractToken(conn, new AuthenticatedURL.Token()); + AuthenticatedURL.extractToken(conn, token); fail(); } catch (AuthenticationException ex) { // Expected + Assert.assertFalse(token.isSet()); } catch (Exception ex) { fail(); } diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 885628871b..351aae0d77 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -261,6 +261,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8509. JarFinder duplicate entry: META-INF/MANIFEST.MF exception (tucu) + HADOOP-8512. AuthenticatedURL should reset the Token when the server returns + other than OK on authentication (tucu) + BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active From 90dade977768b9843e1b36321929de2612933da9 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 18 Jun 2012 16:39:10 +0000 Subject: [PATCH 54/91] SVN-ignoring target/download directories git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351412 13f79535-47bb-0310-9956-ffa450edef68 From 7d40efe915b41f0ca3d007c9c9437a127ba8858a Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Mon, 18 Jun 2012 18:20:41 +0000 Subject: [PATCH 55/91] Revert r1351163 for fixing the JIRA number; it should be HADOOP-8470 but not HADOOP-8468. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351444 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 - .../net/NetworkTopologyWithNodeGroup.java | 398 ------------------ .../src/main/resources/core-default.xml | 12 +- .../net/TestNetworkTopologyWithNodeGroup.java | 165 -------- 4 files changed, 3 insertions(+), 575 deletions(-) delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 351aae0d77..7ed77ed67d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -12,9 +12,6 @@ Trunk (unreleased changes) HADOOP-8469. Make NetworkTopology class pluggable. (Junping Du via szetszwo) - HADOOP-8468. Add NetworkTopologyWithNodeGroup, a 4-layer implementation - of NetworkTopology. (Junping Du via szetszwo) - IMPROVEMENTS HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java deleted file mode 100644 index 6066cd2a61..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java +++ /dev/null @@ -1,398 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.net; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -/** - * The class extends NetworkTopology to represents a cluster of computer with - * a 4-layers hierarchical network topology. - * In this network topology, leaves represent data nodes (computers) and inner - * nodes represent switches/routers that manage traffic in/out of data centers, - * racks or physical host (with virtual switch). - * - * @see NetworkTopology - */ -@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) -@InterfaceStability.Unstable -public class NetworkTopologyWithNodeGroup extends NetworkTopology { - - public final static String DEFAULT_NODEGROUP = "/default-nodegroup"; - - public NetworkTopologyWithNodeGroup() { - clusterMap = new InnerNodeWithNodeGroup(InnerNode.ROOT); - } - - @Override - protected Node getNodeForNetworkLocation(Node node) { - // if node only with default rack info, here we need to add default - // nodegroup info - if (NetworkTopology.DEFAULT_RACK.equals(node.getNetworkLocation())) { - node.setNetworkLocation(node.getNetworkLocation() - + DEFAULT_NODEGROUP); - } - Node nodeGroup = getNode(node.getNetworkLocation()); - if (nodeGroup == null) { - nodeGroup = new InnerNode(node.getNetworkLocation()); - } - return getNode(nodeGroup.getNetworkLocation()); - } - - @Override - public String getRack(String loc) { - netlock.readLock().lock(); - try { - loc = InnerNode.normalize(loc); - Node locNode = getNode(loc); - if (locNode instanceof InnerNodeWithNodeGroup) { - InnerNodeWithNodeGroup node = (InnerNodeWithNodeGroup) locNode; - if (node.isRack()) { - return loc; - } else if (node.isNodeGroup()) { - return node.getNetworkLocation(); - } else { - // may be a data center - return null; - } - } else { - // not in cluster map, don't handle it - return loc; - } - } finally { - netlock.readLock().unlock(); - } - } - - /** - * Given a string representation of a node group for a specific network - * location - * - * @param loc - * a path-like string representation of a network location - * @return a node group string - */ - public String getNodeGroup(String loc) { - netlock.readLock().lock(); - try { - loc = InnerNode.normalize(loc); - Node locNode = getNode(loc); - if (locNode instanceof InnerNodeWithNodeGroup) { - InnerNodeWithNodeGroup node = (InnerNodeWithNodeGroup) locNode; - if (node.isNodeGroup()) { - return loc; - } else if (node.isRack()) { - // not sure the node group for a rack - return null; - } else { - // may be a leaf node - return getNodeGroup(node.getNetworkLocation()); - } - } else { - // not in cluster map, don't handle it - return loc; - } - } finally { - netlock.readLock().unlock(); - } - } - - @Override - public boolean isOnSameRack( Node node1, Node node2) { - if (node1 == null || node2 == null || - node1.getParent() == null || node2.getParent() == null) { - return false; - } - - netlock.readLock().lock(); - try { - return isSameParents(node1.getParent(), node2.getParent()); - } finally { - netlock.readLock().unlock(); - } - } - - /** - * Check if two nodes are on the same node group (hypervisor) The - * assumption here is: each nodes are leaf nodes. - * - * @param node1 - * one node (can be null) - * @param node2 - * another node (can be null) - * @return true if node1 and node2 are on the same node group; false - * otherwise - * @exception IllegalArgumentException - * when either node1 or node2 is null, or node1 or node2 do - * not belong to the cluster - */ - @Override - public boolean isOnSameNodeGroup(Node node1, Node node2) { - if (node1 == null || node2 == null) { - return false; - } - netlock.readLock().lock(); - try { - return isSameParents(node1, node2); - } finally { - netlock.readLock().unlock(); - } - } - - /** - * Check if network topology is aware of NodeGroup - */ - @Override - public boolean isNodeGroupAware() { - return true; - } - - /** Add a leaf node - * Update node counter & rack counter if necessary - * @param node node to be added; can be null - * @exception IllegalArgumentException if add a node to a leave - * or node to be added is not a leaf - */ - @Override - public void add(Node node) { - if (node==null) return; - if( node instanceof InnerNode ) { - throw new IllegalArgumentException( - "Not allow to add an inner node: "+NodeBase.getPath(node)); - } - netlock.writeLock().lock(); - try { - Node rack = null; - - // if node only with default rack info, here we need to add default - // nodegroup info - if (NetworkTopology.DEFAULT_RACK.equals(node.getNetworkLocation())) { - node.setNetworkLocation(node.getNetworkLocation() + - NetworkTopologyWithNodeGroup.DEFAULT_NODEGROUP); - } - Node nodeGroup = getNode(node.getNetworkLocation()); - if (nodeGroup == null) { - nodeGroup = new InnerNodeWithNodeGroup(node.getNetworkLocation()); - } - rack = getNode(nodeGroup.getNetworkLocation()); - - if (rack != null && !(rack instanceof InnerNode)) { - throw new IllegalArgumentException("Unexpected data node " - + node.toString() - + " at an illegal network location"); - } - if (clusterMap.add(node)) { - LOG.info("Adding a new node: " + NodeBase.getPath(node)); - if (rack == null) { - // We only track rack number here - numOfRacks++; - } - } - if(LOG.isDebugEnabled()) { - LOG.debug("NetworkTopology became:\n" + this.toString()); - } - } finally { - netlock.writeLock().unlock(); - } - } - - /** Remove a node - * Update node counter and rack counter if necessary - * @param node node to be removed; can be null - */ - @Override - public void remove(Node node) { - if (node==null) return; - if( node instanceof InnerNode ) { - throw new IllegalArgumentException( - "Not allow to remove an inner node: "+NodeBase.getPath(node)); - } - LOG.info("Removing a node: "+NodeBase.getPath(node)); - netlock.writeLock().lock(); - try { - if (clusterMap.remove(node)) { - Node nodeGroup = getNode(node.getNetworkLocation()); - if (nodeGroup == null) { - nodeGroup = new InnerNode(node.getNetworkLocation()); - } - InnerNode rack = (InnerNode)getNode(nodeGroup.getNetworkLocation()); - if (rack == null) { - numOfRacks--; - } - } - if(LOG.isDebugEnabled()) { - LOG.debug("NetworkTopology became:\n" + this.toString()); - } - } finally { - netlock.writeLock().unlock(); - } - } - - /** Sort nodes array by their distances to reader - * It linearly scans the array, if a local node is found, swap it with - * the first element of the array. - * If a local node group node is found, swap it with the first element - * following the local node. - * If a local rack node is found, swap it with the first element following - * the local node group node. - * If neither local node, node group node or local rack node is found, put a - * random replica location at position 0. - * It leaves the rest nodes untouched. - * @param reader the node that wishes to read a block from one of the nodes - * @param nodes the list of nodes containing data for the reader - */ - @Override - public void pseudoSortByDistance( Node reader, Node[] nodes ) { - - if (reader != null && !this.contains(reader)) { - // if reader is not a datanode (not in NetworkTopology tree), we will - // replace this reader with a sibling leaf node in tree. - Node nodeGroup = getNode(reader.getNetworkLocation()); - if (nodeGroup != null && nodeGroup instanceof InnerNode) { - InnerNode parentNode = (InnerNode) nodeGroup; - // replace reader with the first children of its parent in tree - reader = parentNode.getLeaf(0, null); - } else { - return; - } - } - int tempIndex = 0; - int localRackNode = -1; - int localNodeGroupNode = -1; - if (reader != null) { - //scan the array to find the local node & local rack node - for (int i = 0; i < nodes.length; i++) { - if (tempIndex == 0 && reader == nodes[i]) { //local node - //swap the local node and the node at position 0 - if (i != 0) { - swap(nodes, tempIndex, i); - } - tempIndex=1; - - if (localRackNode != -1 && (localNodeGroupNode !=-1)) { - if (localRackNode == 0) { - localRackNode = i; - } - if (localNodeGroupNode == 0) { - localNodeGroupNode = i; - } - break; - } - } else if (localNodeGroupNode == -1 && isOnSameNodeGroup(reader, - nodes[i])) { - //local node group - localNodeGroupNode = i; - // node local and rack local are already found - if(tempIndex != 0 && localRackNode != -1) break; - } else if (localRackNode == -1 && isOnSameRack(reader, nodes[i])) { - localRackNode = i; - if (tempIndex != 0 && localNodeGroupNode != -1) break; - } - } - - // swap the local nodegroup node and the node at position tempIndex - if(localNodeGroupNode != -1 && localNodeGroupNode != tempIndex) { - swap(nodes, tempIndex, localNodeGroupNode); - if (localRackNode == tempIndex) { - localRackNode = localNodeGroupNode; - } - tempIndex++; - } - - // swap the local rack node and the node at position tempIndex - if(localRackNode != -1 && localRackNode != tempIndex) { - swap(nodes, tempIndex, localRackNode); - tempIndex++; - } - } - - // put a random node at position 0 if there is not a local/local-nodegroup/ - // local-rack node - if (tempIndex == 0 && localNodeGroupNode == -1 && localRackNode == -1 - && nodes.length != 0) { - swap(nodes, 0, r.nextInt(nodes.length)); - } - } - - /** InnerNodeWithNodeGroup represents a switch/router of a data center, rack - * or physical host. Different from a leaf node, it has non-null children. - */ - static class InnerNodeWithNodeGroup extends InnerNode { - public InnerNodeWithNodeGroup(String name, String location, - InnerNode parent, int level) { - super(name, location, parent, level); - } - - public InnerNodeWithNodeGroup(String name, String location) { - super(name, location); - } - - public InnerNodeWithNodeGroup(String path) { - super(path); - } - - @Override - boolean isRack() { - // it is node group - if (getChildren().isEmpty()) { - return false; - } - - Node firstChild = children.get(0); - - if (firstChild instanceof InnerNode) { - Node firstGrandChild = (((InnerNode) firstChild).children).get(0); - if (firstGrandChild instanceof InnerNode) { - // it is datacenter - return false; - } else { - return true; - } - } - return false; - } - - /** - * Judge if this node represents a node group - * - * @return true if it has no child or its children are not InnerNodes - */ - boolean isNodeGroup() { - if (children.isEmpty()) { - return true; - } - Node firstChild = children.get(0); - if (firstChild instanceof InnerNode) { - // it is rack or datacenter - return false; - } - return true; - } - - @Override - protected InnerNode createParentNode(String parentName) { - return new InnerNodeWithNodeGroup(parentName, getPath(this), this, - this.getLevel() + 1); - } - - @Override - protected boolean areChildrenLeaves() { - return isNodeGroup(); - } - } -} diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index c968ff2be6..1e72e362e7 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -599,9 +599,10 @@ - + + - net.topology.node.switch.mapping.impl + net.topology.node.switch.mapping.impl org.apache.hadoop.net.ScriptBasedMapping The default implementation of the DNSToSwitchMapping. It invokes a script specified in net.topology.script.file.name to resolve @@ -610,13 +611,6 @@ - - net.topology.impl - org.apache.hadoop.net.NetworkTopology - The default implementation of NetworkTopology which is classic three layer one. - - - net.topology.script.file.name diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java deleted file mode 100644 index 7dbd33ab95..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.net; - -import java.util.HashMap; -import java.util.Map; - -import junit.framework.TestCase; - -import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; - -public class TestNetworkTopologyWithNodeGroup extends TestCase { - private final static NetworkTopologyWithNodeGroup cluster = new - NetworkTopologyWithNodeGroup(); - - private final static DatanodeDescriptor dataNodes[] = new DatanodeDescriptor[] { - DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/d1/r1/s1"), - DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/d1/r1/s1"), - DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/d1/r1/s2"), - DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/d1/r2/s3"), - DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/d1/r2/s3"), - DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/d1/r2/s4"), - DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/d2/r3/s5"), - DFSTestUtil.getDatanodeDescriptor("8.8.8.8", "/d2/r3/s6") - }; - - private final static NodeBase computeNode = new NodeBase("/d1/r1/s1/h9"); - - static { - for(int i=0; i pickNodesAtRandom(int numNodes, - String excludedScope) { - Map frequency = new HashMap(); - for (DatanodeDescriptor dnd : dataNodes) { - frequency.put(dnd, 0); - } - - for (int j = 0; j < numNodes; j++) { - Node random = cluster.chooseRandom(excludedScope); - frequency.put(random, frequency.get(random) + 1); - } - return frequency; - } - - /** - * This test checks that chooseRandom works for an excluded node. - */ - public void testChooseRandomExcludedNode() { - String scope = "~" + NodeBase.getPath(dataNodes[0]); - Map frequency = pickNodesAtRandom(100, scope); - - for (Node key : dataNodes) { - // all nodes except the first should be more than zero - assertTrue(frequency.get(key) > 0 || key == dataNodes[0]); - } - } - -} From 41f50bc7e8718bb00fe964075e00d63cc532d2d6 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Mon, 18 Jun 2012 18:23:10 +0000 Subject: [PATCH 56/91] HADOOP-8470. Add NetworkTopologyWithNodeGroup, a 4-layer implementation of NetworkTopology. Contributed by Junping Du git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351445 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../net/NetworkTopologyWithNodeGroup.java | 398 ++++++++++++++++++ .../src/main/resources/core-default.xml | 12 +- .../net/TestNetworkTopologyWithNodeGroup.java | 165 ++++++++ 4 files changed, 575 insertions(+), 3 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 7ed77ed67d..ecaa1bf9d2 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -12,6 +12,9 @@ Trunk (unreleased changes) HADOOP-8469. Make NetworkTopology class pluggable. (Junping Du via szetszwo) + HADOOP-8470. Add NetworkTopologyWithNodeGroup, a 4-layer implementation + of NetworkTopology. (Junping Du via szetszwo) + IMPROVEMENTS HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java new file mode 100644 index 0000000000..6066cd2a61 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java @@ -0,0 +1,398 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.net; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * The class extends NetworkTopology to represents a cluster of computer with + * a 4-layers hierarchical network topology. + * In this network topology, leaves represent data nodes (computers) and inner + * nodes represent switches/routers that manage traffic in/out of data centers, + * racks or physical host (with virtual switch). + * + * @see NetworkTopology + */ +@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) +@InterfaceStability.Unstable +public class NetworkTopologyWithNodeGroup extends NetworkTopology { + + public final static String DEFAULT_NODEGROUP = "/default-nodegroup"; + + public NetworkTopologyWithNodeGroup() { + clusterMap = new InnerNodeWithNodeGroup(InnerNode.ROOT); + } + + @Override + protected Node getNodeForNetworkLocation(Node node) { + // if node only with default rack info, here we need to add default + // nodegroup info + if (NetworkTopology.DEFAULT_RACK.equals(node.getNetworkLocation())) { + node.setNetworkLocation(node.getNetworkLocation() + + DEFAULT_NODEGROUP); + } + Node nodeGroup = getNode(node.getNetworkLocation()); + if (nodeGroup == null) { + nodeGroup = new InnerNode(node.getNetworkLocation()); + } + return getNode(nodeGroup.getNetworkLocation()); + } + + @Override + public String getRack(String loc) { + netlock.readLock().lock(); + try { + loc = InnerNode.normalize(loc); + Node locNode = getNode(loc); + if (locNode instanceof InnerNodeWithNodeGroup) { + InnerNodeWithNodeGroup node = (InnerNodeWithNodeGroup) locNode; + if (node.isRack()) { + return loc; + } else if (node.isNodeGroup()) { + return node.getNetworkLocation(); + } else { + // may be a data center + return null; + } + } else { + // not in cluster map, don't handle it + return loc; + } + } finally { + netlock.readLock().unlock(); + } + } + + /** + * Given a string representation of a node group for a specific network + * location + * + * @param loc + * a path-like string representation of a network location + * @return a node group string + */ + public String getNodeGroup(String loc) { + netlock.readLock().lock(); + try { + loc = InnerNode.normalize(loc); + Node locNode = getNode(loc); + if (locNode instanceof InnerNodeWithNodeGroup) { + InnerNodeWithNodeGroup node = (InnerNodeWithNodeGroup) locNode; + if (node.isNodeGroup()) { + return loc; + } else if (node.isRack()) { + // not sure the node group for a rack + return null; + } else { + // may be a leaf node + return getNodeGroup(node.getNetworkLocation()); + } + } else { + // not in cluster map, don't handle it + return loc; + } + } finally { + netlock.readLock().unlock(); + } + } + + @Override + public boolean isOnSameRack( Node node1, Node node2) { + if (node1 == null || node2 == null || + node1.getParent() == null || node2.getParent() == null) { + return false; + } + + netlock.readLock().lock(); + try { + return isSameParents(node1.getParent(), node2.getParent()); + } finally { + netlock.readLock().unlock(); + } + } + + /** + * Check if two nodes are on the same node group (hypervisor) The + * assumption here is: each nodes are leaf nodes. + * + * @param node1 + * one node (can be null) + * @param node2 + * another node (can be null) + * @return true if node1 and node2 are on the same node group; false + * otherwise + * @exception IllegalArgumentException + * when either node1 or node2 is null, or node1 or node2 do + * not belong to the cluster + */ + @Override + public boolean isOnSameNodeGroup(Node node1, Node node2) { + if (node1 == null || node2 == null) { + return false; + } + netlock.readLock().lock(); + try { + return isSameParents(node1, node2); + } finally { + netlock.readLock().unlock(); + } + } + + /** + * Check if network topology is aware of NodeGroup + */ + @Override + public boolean isNodeGroupAware() { + return true; + } + + /** Add a leaf node + * Update node counter & rack counter if necessary + * @param node node to be added; can be null + * @exception IllegalArgumentException if add a node to a leave + * or node to be added is not a leaf + */ + @Override + public void add(Node node) { + if (node==null) return; + if( node instanceof InnerNode ) { + throw new IllegalArgumentException( + "Not allow to add an inner node: "+NodeBase.getPath(node)); + } + netlock.writeLock().lock(); + try { + Node rack = null; + + // if node only with default rack info, here we need to add default + // nodegroup info + if (NetworkTopology.DEFAULT_RACK.equals(node.getNetworkLocation())) { + node.setNetworkLocation(node.getNetworkLocation() + + NetworkTopologyWithNodeGroup.DEFAULT_NODEGROUP); + } + Node nodeGroup = getNode(node.getNetworkLocation()); + if (nodeGroup == null) { + nodeGroup = new InnerNodeWithNodeGroup(node.getNetworkLocation()); + } + rack = getNode(nodeGroup.getNetworkLocation()); + + if (rack != null && !(rack instanceof InnerNode)) { + throw new IllegalArgumentException("Unexpected data node " + + node.toString() + + " at an illegal network location"); + } + if (clusterMap.add(node)) { + LOG.info("Adding a new node: " + NodeBase.getPath(node)); + if (rack == null) { + // We only track rack number here + numOfRacks++; + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("NetworkTopology became:\n" + this.toString()); + } + } finally { + netlock.writeLock().unlock(); + } + } + + /** Remove a node + * Update node counter and rack counter if necessary + * @param node node to be removed; can be null + */ + @Override + public void remove(Node node) { + if (node==null) return; + if( node instanceof InnerNode ) { + throw new IllegalArgumentException( + "Not allow to remove an inner node: "+NodeBase.getPath(node)); + } + LOG.info("Removing a node: "+NodeBase.getPath(node)); + netlock.writeLock().lock(); + try { + if (clusterMap.remove(node)) { + Node nodeGroup = getNode(node.getNetworkLocation()); + if (nodeGroup == null) { + nodeGroup = new InnerNode(node.getNetworkLocation()); + } + InnerNode rack = (InnerNode)getNode(nodeGroup.getNetworkLocation()); + if (rack == null) { + numOfRacks--; + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("NetworkTopology became:\n" + this.toString()); + } + } finally { + netlock.writeLock().unlock(); + } + } + + /** Sort nodes array by their distances to reader + * It linearly scans the array, if a local node is found, swap it with + * the first element of the array. + * If a local node group node is found, swap it with the first element + * following the local node. + * If a local rack node is found, swap it with the first element following + * the local node group node. + * If neither local node, node group node or local rack node is found, put a + * random replica location at position 0. + * It leaves the rest nodes untouched. + * @param reader the node that wishes to read a block from one of the nodes + * @param nodes the list of nodes containing data for the reader + */ + @Override + public void pseudoSortByDistance( Node reader, Node[] nodes ) { + + if (reader != null && !this.contains(reader)) { + // if reader is not a datanode (not in NetworkTopology tree), we will + // replace this reader with a sibling leaf node in tree. + Node nodeGroup = getNode(reader.getNetworkLocation()); + if (nodeGroup != null && nodeGroup instanceof InnerNode) { + InnerNode parentNode = (InnerNode) nodeGroup; + // replace reader with the first children of its parent in tree + reader = parentNode.getLeaf(0, null); + } else { + return; + } + } + int tempIndex = 0; + int localRackNode = -1; + int localNodeGroupNode = -1; + if (reader != null) { + //scan the array to find the local node & local rack node + for (int i = 0; i < nodes.length; i++) { + if (tempIndex == 0 && reader == nodes[i]) { //local node + //swap the local node and the node at position 0 + if (i != 0) { + swap(nodes, tempIndex, i); + } + tempIndex=1; + + if (localRackNode != -1 && (localNodeGroupNode !=-1)) { + if (localRackNode == 0) { + localRackNode = i; + } + if (localNodeGroupNode == 0) { + localNodeGroupNode = i; + } + break; + } + } else if (localNodeGroupNode == -1 && isOnSameNodeGroup(reader, + nodes[i])) { + //local node group + localNodeGroupNode = i; + // node local and rack local are already found + if(tempIndex != 0 && localRackNode != -1) break; + } else if (localRackNode == -1 && isOnSameRack(reader, nodes[i])) { + localRackNode = i; + if (tempIndex != 0 && localNodeGroupNode != -1) break; + } + } + + // swap the local nodegroup node and the node at position tempIndex + if(localNodeGroupNode != -1 && localNodeGroupNode != tempIndex) { + swap(nodes, tempIndex, localNodeGroupNode); + if (localRackNode == tempIndex) { + localRackNode = localNodeGroupNode; + } + tempIndex++; + } + + // swap the local rack node and the node at position tempIndex + if(localRackNode != -1 && localRackNode != tempIndex) { + swap(nodes, tempIndex, localRackNode); + tempIndex++; + } + } + + // put a random node at position 0 if there is not a local/local-nodegroup/ + // local-rack node + if (tempIndex == 0 && localNodeGroupNode == -1 && localRackNode == -1 + && nodes.length != 0) { + swap(nodes, 0, r.nextInt(nodes.length)); + } + } + + /** InnerNodeWithNodeGroup represents a switch/router of a data center, rack + * or physical host. Different from a leaf node, it has non-null children. + */ + static class InnerNodeWithNodeGroup extends InnerNode { + public InnerNodeWithNodeGroup(String name, String location, + InnerNode parent, int level) { + super(name, location, parent, level); + } + + public InnerNodeWithNodeGroup(String name, String location) { + super(name, location); + } + + public InnerNodeWithNodeGroup(String path) { + super(path); + } + + @Override + boolean isRack() { + // it is node group + if (getChildren().isEmpty()) { + return false; + } + + Node firstChild = children.get(0); + + if (firstChild instanceof InnerNode) { + Node firstGrandChild = (((InnerNode) firstChild).children).get(0); + if (firstGrandChild instanceof InnerNode) { + // it is datacenter + return false; + } else { + return true; + } + } + return false; + } + + /** + * Judge if this node represents a node group + * + * @return true if it has no child or its children are not InnerNodes + */ + boolean isNodeGroup() { + if (children.isEmpty()) { + return true; + } + Node firstChild = children.get(0); + if (firstChild instanceof InnerNode) { + // it is rack or datacenter + return false; + } + return true; + } + + @Override + protected InnerNode createParentNode(String parentName) { + return new InnerNodeWithNodeGroup(parentName, getPath(this), this, + this.getLevel() + 1); + } + + @Override + protected boolean areChildrenLeaves() { + return isNodeGroup(); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 1e72e362e7..c968ff2be6 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -599,10 +599,9 @@ - - + - net.topology.node.switch.mapping.impl + net.topology.node.switch.mapping.impl org.apache.hadoop.net.ScriptBasedMapping The default implementation of the DNSToSwitchMapping. It invokes a script specified in net.topology.script.file.name to resolve @@ -611,6 +610,13 @@ + + net.topology.impl + org.apache.hadoop.net.NetworkTopology + The default implementation of NetworkTopology which is classic three layer one. + + + net.topology.script.file.name diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java new file mode 100644 index 0000000000..7dbd33ab95 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.net; + +import java.util.HashMap; +import java.util.Map; + +import junit.framework.TestCase; + +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; + +public class TestNetworkTopologyWithNodeGroup extends TestCase { + private final static NetworkTopologyWithNodeGroup cluster = new + NetworkTopologyWithNodeGroup(); + + private final static DatanodeDescriptor dataNodes[] = new DatanodeDescriptor[] { + DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/d1/r1/s1"), + DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/d1/r1/s1"), + DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/d1/r1/s2"), + DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/d1/r2/s3"), + DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/d1/r2/s3"), + DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/d1/r2/s4"), + DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/d2/r3/s5"), + DFSTestUtil.getDatanodeDescriptor("8.8.8.8", "/d2/r3/s6") + }; + + private final static NodeBase computeNode = new NodeBase("/d1/r1/s1/h9"); + + static { + for(int i=0; i pickNodesAtRandom(int numNodes, + String excludedScope) { + Map frequency = new HashMap(); + for (DatanodeDescriptor dnd : dataNodes) { + frequency.put(dnd, 0); + } + + for (int j = 0; j < numNodes; j++) { + Node random = cluster.chooseRandom(excludedScope); + frequency.put(random, frequency.get(random) + 1); + } + return frequency; + } + + /** + * This test checks that chooseRandom works for an excluded node. + */ + public void testChooseRandomExcludedNode() { + String scope = "~" + NodeBase.getPath(dataNodes[0]); + Map frequency = pickNodesAtRandom(100, scope); + + for (Node key : dataNodes) { + // all nodes except the first should be more than zero + assertTrue(frequency.get(key) > 0 || key == dataNodes[0]); + } + } + +} From 81e449ab8fca20ca41359bd54e8edc0ad6a39eab Mon Sep 17 00:00:00 2001 From: Chun-Yang Chen Date: Tue, 19 Jun 2012 00:55:28 +0000 Subject: [PATCH 57/91] MAPREDUCE-3868. Make Raid Compile. (Weiyan Wang via schen) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351548 13f79535-47bb-0310-9956-ffa450edef68 --- .../resources/assemblies/hadoop-raid-dist.xml | 60 ++++ hadoop-dist/pom.xml | 6 + hadoop-hdfs-project/hadoop-hdfs-raid/pom.xml | 202 ++++++++++++ .../hadoop-hdfs-raid/src/main}/conf/raid.xml | 0 .../hdfs/DistributedRaidFileSystem.java | 0 .../org/apache/hadoop/hdfs/RaidDFSUtil.java | 0 .../BlockPlacementPolicyRaid.java | 0 .../hdfs/server/datanode/RaidBlockSender.java | 294 ++++++++++-------- .../server/namenode/NameNodeRaidUtil.java | 2 +- .../org/apache/hadoop/raid/BlockFixer.java | 17 +- .../org/apache/hadoop/raid/ConfigManager.java | 0 .../java/org/apache/hadoop/raid/Decoder.java | 0 .../hadoop/raid/DirectoryTraversal.java | 16 +- .../apache/hadoop/raid/DistBlockFixer.java | 0 .../java/org/apache/hadoop/raid/DistRaid.java | 1 + .../org/apache/hadoop/raid/DistRaidNode.java | 0 .../java/org/apache/hadoop/raid/Encoder.java | 0 .../org/apache/hadoop/raid/ErasureCode.java | 0 .../org/apache/hadoop/raid/GaloisField.java | 47 ++- .../java/org/apache/hadoop/raid/HarIndex.java | 0 .../org/apache/hadoop/raid/JobMonitor.java | 15 +- .../apache/hadoop/raid/LocalBlockFixer.java | 0 .../org/apache/hadoop/raid/LocalRaidNode.java | 0 .../apache/hadoop/raid/ParityInputStream.java | 0 .../raid/RaidConfigurationException.java | 0 .../org/apache/hadoop/raid/RaidFilter.java | 0 .../java/org/apache/hadoop/raid/RaidNode.java | 11 +- .../org/apache/hadoop/raid/RaidShell.java | 20 +- .../org/apache/hadoop/raid/RaidUtils.java | 0 .../apache/hadoop/raid/ReedSolomonCode.java | 77 +++++ .../hadoop/raid/ReedSolomonDecoder.java | 0 .../hadoop/raid/ReedSolomonEncoder.java | 0 .../org/apache/hadoop/raid/XORDecoder.java | 0 .../org/apache/hadoop/raid/XOREncoder.java | 0 .../hadoop/raid/protocol/PolicyInfo.java | 0 .../hadoop/raid/protocol/PolicyList.java | 0 .../hadoop/raid/protocol/RaidProtocol.java | 0 .../src/main/sbin}/start-raidnode-remote.sh | 0 .../src/main/sbin}/start-raidnode.sh | 0 .../src/main/sbin}/stop-raidnode-remote.sh | 0 .../src/main/sbin}/stop-raidnode.sh | 0 .../org/apache/hadoop/hdfs/TestRaidDfs.java | 5 +- .../TestBlockPlacementPolicyRaid.java | 0 .../server/namenode/NameNodeRaidTestUtil.java | 0 .../apache/hadoop/raid/TestBlockFixer.java | 34 +- .../raid/TestBlockFixerBlockFixDist.java | 0 .../raid/TestBlockFixerDistConcurrency.java | 0 .../TestBlockFixerGeneratedBlockDist.java | 0 .../TestBlockFixerParityBlockFixDist.java | 0 .../hadoop/raid/TestDirectoryTraversal.java | 4 +- .../apache/hadoop/raid/TestErasureCodes.java | 51 +++ .../apache/hadoop/raid/TestGaloisField.java | 0 .../hadoop/raid/TestHarIndexParser.java | 0 .../apache/hadoop/raid/TestRaidFilter.java | 2 +- .../org/apache/hadoop/raid/TestRaidHar.java | 22 +- .../org/apache/hadoop/raid/TestRaidNode.java | 100 ++++-- .../org/apache/hadoop/raid/TestRaidPurge.java | 34 +- .../org/apache/hadoop/raid/TestRaidShell.java | 5 +- .../apache/hadoop/raid/TestRaidShellFsck.java | 8 +- .../hadoop/raid/TestReedSolomonDecoder.java | 2 +- .../hadoop/raid/TestReedSolomonEncoder.java | 2 +- .../hdfs/server/namenode/INodeFile.java | 2 +- hadoop-hdfs-project/pom.xml | 1 + hadoop-mapreduce-project/CHANGES.txt | 2 + hadoop-project/pom.xml | 6 + 65 files changed, 845 insertions(+), 203 deletions(-) create mode 100644 hadoop-assemblies/src/main/resources/assemblies/hadoop-raid-dist.xml create mode 100644 hadoop-hdfs-project/hadoop-hdfs-raid/pom.xml rename {hadoop-mapreduce-project/src/contrib/raid => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/conf/raid.xml (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/hdfs/RaidDFSUtil.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java (63%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidUtil.java (98%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/BlockFixer.java (97%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/ConfigManager.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/Decoder.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/DirectoryTraversal.java (97%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/DistBlockFixer.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/DistRaid.java (99%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/DistRaidNode.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/Encoder.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/ErasureCode.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/GaloisField.java (86%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/HarIndex.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/JobMonitor.java (93%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/LocalBlockFixer.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/LocalRaidNode.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/ParityInputStream.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/RaidConfigurationException.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/RaidFilter.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/RaidNode.java (99%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/RaidShell.java (96%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/RaidUtils.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/ReedSolomonCode.java (54%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/ReedSolomonDecoder.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/ReedSolomonEncoder.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/XORDecoder.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/XOREncoder.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/protocol/PolicyInfo.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/protocol/PolicyList.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src => hadoop-hdfs-project/hadoop-hdfs-raid/src/main}/java/org/apache/hadoop/raid/protocol/RaidProtocol.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/bin => hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin}/start-raidnode-remote.sh (100%) rename {hadoop-mapreduce-project/src/contrib/raid/bin => hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin}/start-raidnode.sh (100%) rename {hadoop-mapreduce-project/src/contrib/raid/bin => hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin}/stop-raidnode-remote.sh (100%) rename {hadoop-mapreduce-project/src/contrib/raid/bin => hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin}/stop-raidnode.sh (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/hdfs/TestRaidDfs.java (99%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockPlacementPolicyRaid.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidTestUtil.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestBlockFixer.java (95%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestBlockFixerBlockFixDist.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestBlockFixerDistConcurrency.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestBlockFixerGeneratedBlockDist.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestBlockFixerParityBlockFixDist.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestDirectoryTraversal.java (98%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestErasureCodes.java (81%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestGaloisField.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestHarIndexParser.java (100%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestRaidFilter.java (98%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestRaidHar.java (92%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestRaidNode.java (88%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestRaidPurge.java (95%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestRaidShell.java (98%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestRaidShellFsck.java (99%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestReedSolomonDecoder.java (98%) rename {hadoop-mapreduce-project/src/contrib/raid/src/test => hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java}/org/apache/hadoop/raid/TestReedSolomonEncoder.java (98%) diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-raid-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-raid-dist.xml new file mode 100644 index 0000000000..a7da364e72 --- /dev/null +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-raid-dist.xml @@ -0,0 +1,60 @@ + + + + hadoop-raid-dist + + dir + + false + + + + ${basedir}/src/main/conf + /etc/hadoop + + * + + + + ${basedir}/src/main/sbin + /sbin + + * + + 0755 + + + ${basedir}/src/main/libexec + /libexec + + * + + 0755 + + + + ${project.build.directory}/site + /share/doc/hadoop/raid + + + + + /share/hadoop/${hadoop.component}/lib + false + runtime + true + + + diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index 7fc6e563db..442cdcac33 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -52,6 +52,11 @@ hadoop-yarn-api provided + + org.apache.hadoop + hadoop-hdfs-raid + provided + @@ -120,6 +125,7 @@ run cp -r $ROOT/hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}/* . run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}/* . run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}/* . + run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-raid/target/hadoop-hdfs-raid-${project.version}/* . run cp -r $ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* . run cp -r $ROOT/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}/* . echo diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-raid/pom.xml new file mode 100644 index 0000000000..f4012b93f2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/pom.xml @@ -0,0 +1,202 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project-dist + 3.0.0-SNAPSHOT + ../../hadoop-project-dist + + org.apache.hadoop + hadoop-hdfs-raid + 3.0.0-SNAPSHOT + jar + + Apache Hadoop HDFS Raid + Apache Hadoop HDFS Raid + + + + raid + false + + + + + junit + junit + test + + + org.apache.hadoop + hadoop-annotations + provided + + + org.apache.hadoop + hadoop-minicluster + test + + + org.apache.hadoop + hadoop-client + provided + + + org.apache.hadoop + hadoop-archives + provided + + + + + + + + maven-dependency-plugin + + + create-mrapp-generated-classpath + generate-test-resources + + build-classpath + + + + ${project.build.directory}/test-classes/mrapp-generated-classpath + + + + + + org.apache.rat + apache-rat-plugin + + + + + + + org.codehaus.mojo + findbugs-maven-plugin + + ${basedir}/dev-support/findbugsExcludeFile.xml + + + + + + + + docs + + false + + + + + org.apache.maven.plugins + maven-site-plugin + + + docs + prepare-package + + site + + + + + + + + + + dist + + false + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.hadoop + hadoop-assemblies + ${project.version} + + + + + dist + package + + single + + + ${project.artifactId}-${project.version} + false + false + + hadoop-raid-dist + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + tar + package + + run + + + + + + + which cygpath 2> /dev/null + if [ $? = 1 ]; then + BUILD_DIR="${project.build.directory}" + else + BUILD_DIR=`cygpath --unix '${project.build.directory}'` + fi + cd $BUILD_DIR + tar czf ${project.artifactId}-${project.version}.tar.gz ${project.artifactId}-${project.version} + + + + + + + + + + + + + + diff --git a/hadoop-mapreduce-project/src/contrib/raid/conf/raid.xml b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/conf/raid.xml similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/conf/raid.xml rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/conf/raid.xml diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/RaidDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/RaidDFSUtil.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/RaidDFSUtil.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/RaidDFSUtil.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java similarity index 63% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java index a29a3ca1b1..7eb6e16c55 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java @@ -34,7 +34,9 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; +import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.net.SocketOutputStream; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.StringUtils; @@ -56,8 +58,10 @@ public class RaidBlockSender implements java.io.Closeable { private DataInputStream checksumIn; // checksum datastream private DataChecksum checksum; // checksum stream private long offset; // starting position to read + /** Initial position to read */ + private long initialOffset; private long endOffset; // ending position - private int bytesPerChecksum; // chunk size + private int chunkSize; // chunk size private int checksumSize; // checksum size private boolean corruptChecksumOk; // if need to verify checksum private boolean chunkOffsetOK; // if need to send chunk offset @@ -74,6 +78,8 @@ public class RaidBlockSender implements java.io.Closeable { * not sure if there will be much more improvement. */ private static final int MIN_BUFFER_WITH_TRANSFERTO = 64*1024; + private static final int TRANSFERTO_BUFFER_SIZE = Math.max( + HdfsConstants.IO_FILE_BUFFER_SIZE, MIN_BUFFER_WITH_TRANSFERTO); private volatile ChunkChecksum lastChunkChecksum = null; @@ -125,12 +131,13 @@ public RaidBlockSender(ExtendedBlock block, long blockLength, long startOffset, * is mostly corrupted. For now just truncate bytesPerchecksum to * blockLength. */ - bytesPerChecksum = checksum.getBytesPerChecksum(); - if (bytesPerChecksum > 10*1024*1024 && bytesPerChecksum > replicaVisibleLength) { + int size = checksum.getBytesPerChecksum(); + if (size > 10*1024*1024 && size > replicaVisibleLength) { checksum = DataChecksum.newDataChecksum(checksum.getChecksumType(), Math.max((int)replicaVisibleLength, 10*1024*1024)); - bytesPerChecksum = checksum.getBytesPerChecksum(); + size = checksum.getBytesPerChecksum(); } + chunkSize = size; checksumSize = checksum.getChecksumSize(); if (length < 0) { @@ -147,12 +154,12 @@ public RaidBlockSender(ExtendedBlock block, long blockLength, long startOffset, throw new IOException(msg); } - offset = (startOffset - (startOffset % bytesPerChecksum)); + offset = (startOffset - (startOffset % chunkSize)); if (length >= 0) { // Make sure endOffset points to end of a checksumed chunk. long tmpLen = startOffset + length; - if (tmpLen % bytesPerChecksum != 0) { - tmpLen += (bytesPerChecksum - tmpLen % bytesPerChecksum); + if (tmpLen % chunkSize != 0) { + tmpLen += (chunkSize - tmpLen % chunkSize); } if (tmpLen < endOffset) { // will use on-disk checksum here since the end is a stable chunk @@ -162,7 +169,7 @@ public RaidBlockSender(ExtendedBlock block, long blockLength, long startOffset, // seek to the right offsets if (offset > 0) { - long checksumSkip = (offset / bytesPerChecksum) * checksumSize; + long checksumSkip = (offset / chunkSize) * checksumSize; // note blockInStream is seeked when created below if (checksumSkip > 0) { // Should we use seek() for checksum file as well? @@ -178,7 +185,7 @@ public RaidBlockSender(ExtendedBlock block, long blockLength, long startOffset, throw ioe; } } - + /** * close opened files. */ @@ -227,57 +234,85 @@ private static IOException ioeToSocketException(IOException ioe) { // otherwise just return the same exception. return ioe; } - + /** - * Sends upto maxChunks chunks of data. - * - * When blockInPosition is >= 0, assumes 'out' is a - * {@link SocketOutputStream} and tries - * {@link SocketOutputStream#transferToFully(FileChannel, long, int)} to - * send data (and updates blockInPosition). + * @param datalen Length of data + * @return number of chunks for data of given size */ - private int sendChunks(ByteBuffer pkt, int maxChunks, OutputStream out) - throws IOException { - // Sends multiple chunks in one packet with a single write(). - - int len = (int) Math.min(endOffset - offset, - (((long) bytesPerChecksum) * ((long) maxChunks))); - int numChunks = (len + bytesPerChecksum - 1)/bytesPerChecksum; - int packetLen = len + numChunks*checksumSize + 4; - boolean lastDataPacket = offset + len == endOffset && len > 0; + private int numberOfChunks(long datalen) { + return (int) ((datalen + chunkSize - 1)/chunkSize); + } + + /** + * Write packet header into {@code pkt} + */ + private void writePacketHeader(ByteBuffer pkt, int dataLen, int packetLen) { pkt.clear(); - - - PacketHeader header = new PacketHeader( - packetLen, offset, seqno, (len == 0), len); + PacketHeader header = new PacketHeader(packetLen, offset, seqno, + (dataLen == 0), dataLen, false); header.putInBuffer(pkt); + } + + /** + * Read checksum into given buffer + * @param buf buffer to read the checksum into + * @param checksumOffset offset at which to write the checksum into buf + * @param checksumLen length of checksum to write + * @throws IOException on error + */ + private void readChecksum(byte[] buf, final int checksumOffset, + final int checksumLen) throws IOException { + if (checksumSize <= 0 && checksumIn == null) { + return; + } + try { + checksumIn.readFully(buf, checksumOffset, checksumLen); + } catch (IOException e) { + LOG.warn(" Could not read or failed to veirfy checksum for data" + + " at offset " + offset + " for block " + block, e); + IOUtils.closeStream(checksumIn); + checksumIn = null; + if (corruptChecksumOk) { + if (checksumOffset < checksumLen) { + // Just fill the array with zeros. + Arrays.fill(buf, checksumOffset, checksumLen, (byte) 0); + } + } else { + throw e; + } + } + } + + /** + * Sends a packet with up to maxChunks chunks of data. + * + * @param pkt buffer used for writing packet data + * @param maxChunks maximum number of chunks to send + * @param out stream to send data to + * @param transferTo use transferTo to send data + * @param throttler used for throttling data transfer bandwidth + */ + private int sendPacket(ByteBuffer pkt, int maxChunks, OutputStream out, + boolean transferTo, DataTransferThrottler throttler) throws IOException { + int dataLen = (int) Math.min(endOffset - offset, + (chunkSize * (long) maxChunks)); + + int numChunks = numberOfChunks(dataLen); // Number of chunks be sent in the packet + int checksumDataLen = numChunks * checksumSize; + int packetLen = dataLen + checksumDataLen + 4; + boolean lastDataPacket = offset + dataLen == endOffset && dataLen > 0; + + writePacketHeader(pkt, dataLen, packetLen); int checksumOff = pkt.position(); - int checksumLen = numChunks * checksumSize; byte[] buf = pkt.array(); if (checksumSize > 0 && checksumIn != null) { - try { - checksumIn.readFully(buf, checksumOff, checksumLen); - } catch (IOException e) { - LOG.warn(" Could not read or failed to veirfy checksum for data" + - " at offset " + offset + " for block " + block + " got : " - + StringUtils.stringifyException(e)); - IOUtils.closeStream(checksumIn); - checksumIn = null; - if (corruptChecksumOk) { - if (checksumOff < checksumLen) { - // Just fill the array with zeros. - Arrays.fill(buf, checksumOff, checksumLen, (byte) 0); - } - } else { - throw e; - } - } + readChecksum(buf, checksumOff, checksumDataLen); // write in progress that we need to use to get last checksum if (lastDataPacket && lastChunkChecksum != null) { - int start = checksumOff + checksumLen - checksumSize; + int start = checksumOff + checksumDataLen - checksumSize; byte[] updatedChecksum = lastChunkChecksum.getChecksum(); if (updatedChecksum != null) { @@ -286,61 +321,85 @@ private int sendChunks(ByteBuffer pkt, int maxChunks, OutputStream out) } } - int dataOff = checksumOff + checksumLen; - - if (blockInPosition < 0) { - //normal transfer - IOUtils.readFully(blockIn, buf, dataOff, len); + int dataOff = checksumOff + checksumDataLen; + if (!transferTo) { // normal transfer + IOUtils.readFully(blockIn, buf, dataOff, dataLen); if (verifyChecksum) { - int dOff = dataOff; - int cOff = checksumOff; - int dLeft = len; - - for (int i=0; i= 0) { - //use transferTo(). Checks on out and blockIn are already done. - + if (transferTo) { SocketOutputStream sockOut = (SocketOutputStream)out; - //first write the packet - sockOut.write(buf, 0, dataOff); + sockOut.write(buf, 0, dataOff); // First write checksum + // no need to flush. since we know out is not a buffered stream. - sockOut.transferToFully(((FileInputStream)blockIn).getChannel(), - blockInPosition, len); - - blockInPosition += len; - } else { + blockInPosition, dataLen); + blockInPosition += dataLen; + } else { // normal transfer - out.write(buf, 0, dataOff + len); + out.write(buf, 0, dataOff + dataLen); } - } catch (IOException e) { - /* exception while writing to the client (well, with transferTo(), - * it could also be while reading from the local file). + /* Exception while writing to the client. Connection closure from + * the other end is mostly the case and we do not care much about + * it. But other things can go wrong, especially in transferTo(), + * which we do not want to ignore. + * + * The message parsing below should not be considered as a good + * coding example. NEVER do it to drive a program logic. NEVER. + * It was done here because the NIO throws an IOException for EPIPE. */ + String ioem = e.getMessage(); + if (!ioem.startsWith("Broken pipe") && !ioem.startsWith("Connection reset")) { + LOG.error("BlockSender.sendChunks() exception: ", e); + } throw ioeToSocketException(e); } - return len; + if (throttler != null) { // rebalancing so throttle + throttler.throttle(packetLen); + } + + return dataLen; } + + /** + * Compute checksum for chunks and verify the checksum that is read from + * the metadata file is correct. + * + * @param buf buffer that has checksum and data + * @param dataOffset position where data is written in the buf + * @param datalen length of data + * @param numChunks number of chunks corresponding to data + * @param checksumOffset offset where checksum is written in the buf + * @throws ChecksumException on failed checksum verification + */ + public void verifyChecksum(final byte[] buf, final int dataOffset, + final int datalen, final int numChunks, final int checksumOffset) + throws ChecksumException { + int dOff = dataOffset; + int cOff = checksumOffset; + int dLeft = datalen; + + for (int i = 0; i < numChunks; i++) { + checksum.reset(); + int dLen = Math.min(dLeft, chunkSize); + checksum.update(buf, dOff, dLen); + if (!checksum.compare(buf, cOff)) { + long failedPos = offset + datalen - dLeft; + throw new ChecksumException("Checksum failed at " + failedPos, + failedPos); + } + dLeft -= dLen; + dOff += dLen; + cOff += checksumSize; + } + } + /** * sendBlock() is used to read block and its metadata and stream the data to @@ -356,79 +415,61 @@ private int sendChunks(ByteBuffer pkt, int maxChunks, OutputStream out) */ public long sendBlock(DataOutputStream out, OutputStream baseStream) throws IOException { - if( out == null ) { + if (out == null) { throw new IOException( "out stream is null" ); } - - long initialOffset = offset; + initialOffset = offset; long totalRead = 0; OutputStream streamForSendChunks = out; final long startTime = ClientTraceLog.isInfoEnabled() ? System.nanoTime() : 0; try { - try { - checksum.writeHeader(out); - if ( chunkOffsetOK ) { - out.writeLong( offset ); - } - out.flush(); - } catch (IOException e) { //socket error - throw ioeToSocketException(e); - } - int maxChunksPerPacket; int pktSize = PacketHeader.PKT_HEADER_LEN; - - if (transferToAllowed && !verifyChecksum && - baseStream instanceof SocketOutputStream && - blockIn instanceof FileInputStream) { - + boolean transferTo = transferToAllowed && !verifyChecksum + && baseStream instanceof SocketOutputStream + && blockIn instanceof FileInputStream; + if (transferTo) { FileChannel fileChannel = ((FileInputStream)blockIn).getChannel(); - - // blockInPosition also indicates sendChunks() uses transferTo. blockInPosition = fileChannel.position(); streamForSendChunks = baseStream; + maxChunksPerPacket = numberOfChunks(TRANSFERTO_BUFFER_SIZE); - // assure a mininum buffer size. - maxChunksPerPacket = (Math.max(HdfsConstants.IO_FILE_BUFFER_SIZE, - MIN_BUFFER_WITH_TRANSFERTO) - + bytesPerChecksum - 1)/bytesPerChecksum; - - // allocate smaller buffer while using transferTo(). + // Smaller packet size to only hold checksum when doing transferTo pktSize += checksumSize * maxChunksPerPacket; } else { maxChunksPerPacket = Math.max(1, - (HdfsConstants.IO_FILE_BUFFER_SIZE + bytesPerChecksum - 1)/bytesPerChecksum); - pktSize += (bytesPerChecksum + checksumSize) * maxChunksPerPacket; + numberOfChunks(HdfsConstants.IO_FILE_BUFFER_SIZE)); + // Packet size includes both checksum and data + pktSize += (chunkSize + checksumSize) * maxChunksPerPacket; } ByteBuffer pktBuf = ByteBuffer.allocate(pktSize); while (endOffset > offset) { - long len = sendChunks(pktBuf, maxChunksPerPacket, - streamForSendChunks); + long len = sendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, + transferTo, null); offset += len; - totalRead += len + ((len + bytesPerChecksum - 1)/bytesPerChecksum* - checksumSize); + totalRead += len + (numberOfChunks(len) * checksumSize); seqno++; } try { // send an empty packet to mark the end of the block - sendChunks(pktBuf, maxChunksPerPacket, streamForSendChunks); + sendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, transferTo, + null); out.flush(); } catch (IOException e) { //socket error throw ioeToSocketException(e); } + blockReadFully = true; } finally { if (clientTraceFmt != null) { final long endTime = System.nanoTime(); - ClientTraceLog.info(String.format(clientTraceFmt, totalRead, initialOffset, endTime - startTime)); + ClientTraceLog.info(String.format(clientTraceFmt, totalRead, + initialOffset, endTime - startTime)); } close(); } - - blockReadFully = initialOffset == 0 && offset >= replicaVisibleLength; - return totalRead; } @@ -440,6 +481,13 @@ public static interface InputStreamFactory { public InputStream createStream(long offset) throws IOException; } + /** + * @return the checksum type that will be used with this block transfer. + */ + public DataChecksum getChecksum() { + return checksum; + } + private static class BlockInputStreamFactory implements InputStreamFactory { private final ExtendedBlock block; private final FsDatasetSpi data; diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidUtil.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidUtil.java similarity index 98% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidUtil.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidUtil.java index 531a0f238e..9258696156 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidUtil.java @@ -50,7 +50,7 @@ public static LocatedBlocks getBlockLocations(final FSNamesystem namesystem, final boolean doAccessTime, final boolean needBlockToken ) throws FileNotFoundException, UnresolvedLinkException, IOException { return namesystem.getBlockLocations(src, offset, length, - doAccessTime, needBlockToken); + doAccessTime, needBlockToken, true); } } diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/BlockFixer.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/BlockFixer.java similarity index 97% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/BlockFixer.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/BlockFixer.java index 6e1d7f7917..dd2fb96c2b 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/BlockFixer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/BlockFixer.java @@ -18,6 +18,9 @@ package org.apache.hadoop.raid; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY; + import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -44,14 +47,17 @@ import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.datatransfer.*; +import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader; +import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.RaidBlockSender; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -61,6 +67,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.RaidDFSUtil; +import org.apache.hadoop.io.Text; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.net.NetUtils; @@ -649,7 +656,7 @@ static DataInputStream computeMetadata(Configuration conf, mdOut.writeShort(BlockMetadataHeader.VERSION); // Create a summer and write out its header. - int bytesPerChecksum = conf.getInt("io.bytes.per.checksum", 512); + int bytesPerChecksum = conf.getInt("dfs.bytes-per-checksum", 512); DataChecksum sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, bytesPerChecksum); @@ -709,8 +716,8 @@ private void computeMetadataAndSendFixedBlock(DatanodeInfo datanode, blockContents.close(); // Reopen blockContents = new FileInputStream(localBlockFile); - sendFixedBlock(datanode, blockContents, blockMetadata, block, - blockSize); + sendFixedBlock(datanode, blockContents, blockMetadata, block, + blockSize); } finally { if (blockContents != null) { blockContents.close(); @@ -780,9 +787,11 @@ private void sendFixedBlock(DatanodeInfo datanode, }); DatanodeInfo[] nodes = new DatanodeInfo[]{datanode}; + DataChecksum checksum = blockSender.getChecksum(); new Sender(out).writeBlock(block.getBlock(), block.getBlockToken(), "", nodes, null, BlockConstructionStage.PIPELINE_SETUP_CREATE, - 1, 0L, blockSize, 0L, DataChecksum.newDataChecksum(metadataIn)); + 1, 0L, blockSize, 0L, DataChecksum.newDataChecksum( + checksum.getChecksumType(), checksum.getBytesPerChecksum())); blockSender.sendBlock(out, baseStream); LOG.info("Sent block " + block.getBlock() + " to " + datanode.getName()); diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ConfigManager.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ConfigManager.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/Decoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Decoder.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/Decoder.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Decoder.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DirectoryTraversal.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DirectoryTraversal.java similarity index 97% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DirectoryTraversal.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DirectoryTraversal.java index 4c955df005..c1ff9bccc4 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DirectoryTraversal.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DirectoryTraversal.java @@ -112,6 +112,8 @@ public DirectoryTraversal( public List getFilteredFiles(FileFilter filter, int limit) { List filtered = new ArrayList(); + if (limit == 0) + return filtered; // We need this semaphore to block when the number of running workitems // is equal to the number of threads. FixedThreadPool limits the number @@ -120,20 +122,26 @@ public List getFilteredFiles(FileFilter filter, int limit) { Semaphore slots = new Semaphore(numThreads); while (true) { - synchronized(filtered) { - if (filtered.size() >= limit) break; - } FilterFileWorkItem work = null; try { + slots.acquire(); + synchronized(filtered) { + if (filtered.size() >= limit) { + slots.release(); + break; + } + } Node next = getNextDirectoryNode(); if (next == null) { + slots.release(); break; } work = new FilterFileWorkItem(filter, next, filtered, slots); - slots.acquire(); } catch (InterruptedException ie) { + slots.release(); break; } catch (IOException e) { + slots.release(); break; } executor.execute(work); diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistBlockFixer.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistBlockFixer.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaid.java similarity index 99% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaid.java index 81a3198f57..5c6e5cc79d 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaid.java @@ -277,6 +277,7 @@ public boolean startDistRaid() throws IOException { */ public boolean checkComplete() throws IOException { JobID jobID = runningJob.getJobID(); + LOG.info("Checking job " + jobID); try { if (runningJob.isComplete()) { // delete job directory diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaidNode.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaidNode.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaidNode.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaidNode.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/Encoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Encoder.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/Encoder.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Encoder.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/ErasureCode.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ErasureCode.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/ErasureCode.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ErasureCode.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/GaloisField.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/GaloisField.java similarity index 86% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/GaloisField.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/GaloisField.java index 78b7af1b34..a10f5d79ea 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/GaloisField.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/GaloisField.java @@ -208,7 +208,7 @@ public void solveVandermondeSystem(int[] x, int[] y) { * @param len consider x and y only from 0...len-1 */ public void solveVandermondeSystem(int[] x, int[] y, int len) { - assert(x.length <= len && y.length <= len); + assert(y.length <= len); for (int i = 0; i < len - 1; i++) { for (int j = len - 1; j > i; j--) { y[j] = y[j] ^ mulTable[x[i]][y[j - 1]]; @@ -302,4 +302,49 @@ public int substitute(int[] p, int x) { } return result; } + + /** + * Perform Gaussian elimination on the given matrix. This matrix has to be a + * fat matrix (number of rows > number of columns). + */ + public void gaussianElimination(int[][] matrix) { + assert(matrix != null && matrix.length > 0 && matrix[0].length > 0 + && matrix.length < matrix[0].length); + int height = matrix.length; + int width = matrix[0].length; + for (int i = 0; i < height; i++) { + boolean pivotFound = false; + // scan the column for a nonzero pivot and swap it to the diagonal + for (int j = i; j < height; j++) { + if (matrix[i][j] != 0) { + int[] tmp = matrix[i]; + matrix[i] = matrix[j]; + matrix[j] = tmp; + pivotFound = true; + break; + } + } + if (!pivotFound) { + continue; + } + int pivot = matrix[i][i]; + for (int j = i; j < width; j++) { + matrix[i][j] = divide(matrix[i][j], pivot); + } + for (int j = i + 1; j < height; j++) { + int lead = matrix[j][i]; + for (int k = i; k < width; k++) { + matrix[j][k] = add(matrix[j][k], multiply(lead, matrix[i][k])); + } + } + } + for (int i = height - 1; i >=0; i--) { + for (int j = 0; j < i; j++) { + int lead = matrix[j][i]; + for (int k = i; k < width; k++) { + matrix[j][k] = add(matrix[j][k], multiply(lead, matrix[i][k])); + } + } + } + } } diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/HarIndex.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/HarIndex.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/HarIndex.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/HarIndex.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/JobMonitor.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/JobMonitor.java similarity index 93% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/JobMonitor.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/JobMonitor.java index e01fcba73e..cda295c65a 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/JobMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/JobMonitor.java @@ -44,12 +44,13 @@ class JobMonitor implements Runnable { volatile boolean running = true; private Map> jobs; + public static final String JOBMONITOR_INTERVAL_KEY = "raid.jobmonitor.interval"; private long jobMonitorInterval; private volatile long jobsMonitored = 0; private volatile long jobsSucceeded = 0; public JobMonitor(Configuration conf) { - jobMonitorInterval = conf.getLong("raid.jobmonitor.interval", 60000); + jobMonitorInterval = conf.getLong(JOBMONITOR_INTERVAL_KEY, 60000); jobs = new java.util.HashMap>(); } @@ -112,6 +113,7 @@ public void doMonitor() { } catch (IOException ioe) { // If there was an error, consider the job finished. addJob(finishedJobs, key, job); + LOG.error("JobMonitor exception", ioe); } } } @@ -159,6 +161,17 @@ public long jobsMonitored() { public long jobsSucceeded() { return this.jobsSucceeded; } + + // For test code + int runningJobsCount() { + int total = 0; + synchronized(jobs) { + for (String key: jobs.keySet()) { + total += jobs.get(key).size(); + } + } + return total; + } private static void addJob(Map> jobsMap, String jobName, DistRaid job) { diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/LocalBlockFixer.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/LocalBlockFixer.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/LocalBlockFixer.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/LocalBlockFixer.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/LocalRaidNode.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/LocalRaidNode.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/LocalRaidNode.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/LocalRaidNode.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/ParityInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ParityInputStream.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/ParityInputStream.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ParityInputStream.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidConfigurationException.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidConfigurationException.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidConfigurationException.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidConfigurationException.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidFilter.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidFilter.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidFilter.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidFilter.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidNode.java similarity index 99% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidNode.java index 48329d357f..dc196494cf 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidNode.java @@ -80,6 +80,8 @@ public abstract class RaidNode implements RaidProtocol { } public static final Log LOG = LogFactory.getLog( "org.apache.hadoop.raid.RaidNode"); public static final long SLEEP_TIME = 10000L; // 10 seconds + public static final String TRIGGER_MONITOR_SLEEP_TIME_KEY = + "hdfs.raid.trigger.monitor.sleep.time"; public static final int DEFAULT_PORT = 60000; // Default stripe length = 5, parity length for RS code = 3 public static final int DEFAULT_STRIPE_LENGTH = 5; @@ -126,6 +128,7 @@ public abstract class RaidNode implements RaidProtocol { /** Deamon thread to trigger policies */ Daemon triggerThread = null; + public static long triggerMonitorSleepTime = SLEEP_TIME; /** Deamon thread to delete obsolete parity files */ PurgeMonitor purgeMonitor = null; @@ -299,6 +302,10 @@ private void initialize(Configuration conf) this.blockFixer = BlockFixer.createBlockFixer(conf); this.blockFixerThread = new Daemon(this.blockFixer); this.blockFixerThread.start(); + // start the deamon thread to fire polcies appropriately + RaidNode.triggerMonitorSleepTime = conf.getLong( + TRIGGER_MONITOR_SLEEP_TIME_KEY, + SLEEP_TIME); // start the deamon thread to fire polcies appropriately this.triggerThread = new Daemon(new TriggerMonitor()); @@ -503,7 +510,7 @@ private void doProcess() throws IOException, InterruptedException { } } while (running) { - Thread.sleep(SLEEP_TIME); + Thread.sleep(RaidNode.triggerMonitorSleepTime); boolean reloaded = configMgr.reloadConfigsIfNecessary(); if (reloaded) { @@ -542,7 +549,7 @@ private void doProcess() throws IOException, InterruptedException { // Apply the action on accepted paths LOG.info("Triggering Policy Action " + info.getName() + - " " + info.getSrcPath()); + " " + info.getSrcPath() + " raid " + filteredPaths.size() + " files"); try { raidFiles(info, filteredPaths); } catch (Exception e) { diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidShell.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidShell.java similarity index 96% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidShell.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidShell.java index 479043caad..55812101ca 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidShell.java @@ -43,6 +43,7 @@ import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryProxy; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -296,9 +297,22 @@ public Path[] recover(String cmd, String argv[], int startindex) for (int i = startindex; i < argv.length; i = i + 2) { String path = argv[i]; long corruptOffset = Long.parseLong(argv[i+1]); - LOG.debug("RaidShell recoverFile for " + path + " corruptOffset " + corruptOffset); - paths[j] = new Path(raidnode.recoverFile(path, corruptOffset)); - LOG.debug("Raidshell created recovery file " + paths[j]); + LOG.info("RaidShell recoverFile for " + path + " corruptOffset " + corruptOffset); + Path recovered = new Path("/tmp/recovered." + System.currentTimeMillis()); + FileSystem fs = recovered.getFileSystem(conf); + DistributedFileSystem dfs = (DistributedFileSystem)fs; + Configuration raidConf = new Configuration(conf); + raidConf.set("fs.hdfs.impl", + "org.apache.hadoop.hdfs.DistributedRaidFileSystem"); + raidConf.set("fs.raid.underlyingfs.impl", + "org.apache.hadoop.hdfs.DistributedFileSystem"); + raidConf.setBoolean("fs.hdfs.impl.disable.cache", true); + java.net.URI dfsUri = dfs.getUri(); + FileSystem raidFs = FileSystem.get(dfsUri, raidConf); + FileUtil.copy(raidFs, new Path(path), fs, recovered, false, conf); + + paths[j] = recovered; + LOG.info("Raidshell created recovery file " + paths[j]); j++; } return paths; diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidUtils.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidUtils.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidUtils.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidUtils.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/ReedSolomonCode.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonCode.java similarity index 54% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/ReedSolomonCode.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonCode.java index 639e537369..0a5d91bf4d 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/ReedSolomonCode.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonCode.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.raid; +import java.util.Set; + public class ReedSolomonCode implements ErasureCode { @@ -103,4 +105,79 @@ public int paritySize() { public int symbolSize() { return (int) Math.round(Math.log(GF.getFieldSize()) / Math.log(2)); } + + /** + * Given parity symbols followed by message symbols, return the locations of + * symbols that are corrupted. Can resolve up to (parity length / 2) error + * locations. + * @param data The message and parity. The parity should be placed in the + * first part of the array. In each integer, the relevant portion + * is present in the least significant bits of each int. + * The number of elements in data is stripeSize() + paritySize(). + * Note that data may be changed after calling this method. + * @param errorLocations The set to put the error location results + * @return true If the locations can be resolved, return true. + */ + public boolean computeErrorLocations(int[] data, + Set errorLocations) { + assert(data.length == paritySize + stripeSize && errorLocations != null); + errorLocations.clear(); + int maxError = paritySize / 2; + int[][] syndromeMatrix = new int[maxError][]; + for (int i = 0; i < syndromeMatrix.length; ++i) { + syndromeMatrix[i] = new int[maxError + 1]; + } + int[] syndrome = new int[paritySize]; + + if (computeSyndrome(data, syndrome)) { + // Parity check OK. No error location added. + return true; + } + for (int i = 0; i < maxError; ++i) { + for (int j = 0; j < maxError + 1; ++j) { + syndromeMatrix[i][j] = syndrome[i + j]; + } + } + GF.gaussianElimination(syndromeMatrix); + int[] polynomial = new int[maxError + 1]; + polynomial[0] = 1; + for (int i = 0; i < maxError; ++i) { + polynomial[i + 1] = syndromeMatrix[maxError - 1 - i][maxError]; + } + for (int i = 0; i < paritySize + stripeSize; ++i) { + int possibleRoot = GF.divide(1, primitivePower[i]); + if (GF.substitute(polynomial, possibleRoot) == 0) { + errorLocations.add(i); + } + } + // Now recover with error locations and check the syndrome again + int[] locations = new int[errorLocations.size()]; + int k = 0; + for (int loc : errorLocations) { + locations[k++] = loc; + } + int [] erasedValue = new int[locations.length]; + decode(data, locations, erasedValue); + for (int i = 0; i < locations.length; ++i) { + data[locations[i]] = erasedValue[i]; + } + return computeSyndrome(data, syndrome); + } + + /** + * Compute the syndrome of the input [parity, message] + * @param data [parity, message] + * @param syndrome The syndromes (checksums) of the data + * @return true If syndromes are all zeros + */ + private boolean computeSyndrome(int[] data, int [] syndrome) { + boolean corruptionFound = false; + for (int i = 0; i < paritySize; i++) { + syndrome[i] = GF.substitute(data, primitivePower[i]); + if (syndrome[i] != 0) { + corruptionFound = true; + } + } + return !corruptionFound; + } } diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/ReedSolomonDecoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonDecoder.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/ReedSolomonDecoder.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonDecoder.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/ReedSolomonEncoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonEncoder.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/ReedSolomonEncoder.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonEncoder.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/XORDecoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/XORDecoder.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/XORDecoder.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/XORDecoder.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/XOREncoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/XOREncoder.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/XOREncoder.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/XOREncoder.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/protocol/PolicyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/PolicyInfo.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/protocol/PolicyInfo.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/PolicyInfo.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/protocol/PolicyList.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/PolicyList.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/protocol/PolicyList.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/PolicyList.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/protocol/RaidProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/RaidProtocol.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/protocol/RaidProtocol.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/RaidProtocol.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/bin/start-raidnode-remote.sh b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/start-raidnode-remote.sh similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/bin/start-raidnode-remote.sh rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/start-raidnode-remote.sh diff --git a/hadoop-mapreduce-project/src/contrib/raid/bin/start-raidnode.sh b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/start-raidnode.sh similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/bin/start-raidnode.sh rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/start-raidnode.sh diff --git a/hadoop-mapreduce-project/src/contrib/raid/bin/stop-raidnode-remote.sh b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/stop-raidnode-remote.sh similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/bin/stop-raidnode-remote.sh rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/stop-raidnode-remote.sh diff --git a/hadoop-mapreduce-project/src/contrib/raid/bin/stop-raidnode.sh b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/stop-raidnode.sh similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/bin/stop-raidnode.sh rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/stop-raidnode.sh diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/hdfs/TestRaidDfs.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/TestRaidDfs.java similarity index 99% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/hdfs/TestRaidDfs.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/TestRaidDfs.java index ffdb4edfa0..36aab52ee5 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/hdfs/TestRaidDfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/TestRaidDfs.java @@ -47,8 +47,8 @@ public class TestRaidDfs extends TestCase { final static String TEST_DIR = new File(System.getProperty("test.build.data", - "build/contrib/raid/test/data")).getAbsolutePath(); - final static String LOG_DIR = "/raidlog"; + "target/test-data")).getAbsolutePath(); + final static String LOG_DIR = "target/raidlog"; final static long RELOAD_INTERVAL = 1000; final static Log LOG = LogFactory.getLog("org.apache.hadoop.raid.TestRaidDfs"); final static int NUM_DATANODES = 3; @@ -414,6 +414,7 @@ public static boolean validateFile(FileSystem fileSys, Path name, long length, LOG.info(" Newcrc " + newcrc.getValue() + " old crc " + crc); if (newcrc.getValue() != crc) { LOG.info("CRC mismatch of file " + name + ": " + newcrc + " vs. " + crc); + return false; } return true; } diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockPlacementPolicyRaid.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockPlacementPolicyRaid.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockPlacementPolicyRaid.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockPlacementPolicyRaid.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidTestUtil.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidTestUtil.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidTestUtil.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestBlockFixer.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixer.java similarity index 95% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestBlockFixer.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixer.java index 10a7212645..8986bea8da 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestBlockFixer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixer.java @@ -26,12 +26,14 @@ import java.util.Random; import java.util.zip.CRC32; +import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.junit.Test; import static org.junit.Assert.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.JarFinder; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -39,6 +41,8 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.JobContext; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -53,9 +57,11 @@ public class TestBlockFixer { final static Log LOG = LogFactory.getLog( "org.apache.hadoop.raid.TestBlockFixer"); final static String TEST_DIR = new File(System.getProperty("test.build.data", - "build/contrib/raid/test/data")).getAbsolutePath(); + "target/test-data")).getAbsolutePath(); final static String CONFIG_FILE = new File(TEST_DIR, "test-raid.xml").getAbsolutePath(); + public static final String DistBlockFixer_JAR = + JarFinder.getJar(DistBlockFixer.class); final static long RELOAD_INTERVAL = 1000; final static int NUM_DATANODES = 3; Configuration conf; @@ -546,6 +552,8 @@ protected void mySetup(int stripeLength, int timeBeforeHar) throws Exception { conf.setBoolean("dfs.permissions", false); + conf.set("mapreduce.framework.name", "yarn"); + dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build(); dfs.waitActive(); fileSys = dfs.getFileSystem(); @@ -553,11 +561,28 @@ protected void mySetup(int stripeLength, int timeBeforeHar) throws Exception { FileSystem.setDefaultUri(conf, namenode); mr = new MiniMRCluster(4, namenode, 3); - jobTrackerName = "localhost:" + mr.getJobTrackerPort(); + JobConf jobConf = mr.createJobConf(); + jobTrackerName = "localhost:" + jobConf.get(JTConfig.JT_IPC_ADDRESS); hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); FileSystem.setDefaultUri(conf, namenode); conf.set("mapred.job.tracker", jobTrackerName); + conf.set("mapreduce.framework.name", "yarn"); + String rmAdress = jobConf.get("yarn.resourcemanager.address"); + if (rmAdress != null) { + conf.set("yarn.resourcemanager.address", rmAdress); + } + String schedulerAdress = + jobConf.get("yarn.resourcemanager.scheduler.address"); + if (schedulerAdress != null) { + conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress); + } + String jobHistoryAddress = + jobConf.get("mapreduce.jobhistory.address"); + if (jobHistoryAddress != null) { + conf.set("mapreduce.jobhistory.address", jobHistoryAddress); + } + conf.set(JobContext.JAR, TestBlockFixer.DistBlockFixer_JAR); FileWriter fileWriter = new FileWriter(CONFIG_FILE); fileWriter.write("\n"); @@ -609,10 +634,11 @@ protected void myTearDown() throws Exception { if (dfs != null) { dfs.shutdown(); } } - private long getCRC(FileSystem fs, Path p) throws IOException { + public static long getCRC(FileSystem fs, Path p) throws IOException { CRC32 crc = new CRC32(); FSDataInputStream stm = fs.open(p); - for (int b = 0; b > 0; b = stm.read()) { + int b; + while ((b = stm.read())>=0) { crc.update(b); } stm.close(); diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestBlockFixerBlockFixDist.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerBlockFixDist.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestBlockFixerBlockFixDist.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerBlockFixDist.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestBlockFixerDistConcurrency.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerDistConcurrency.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestBlockFixerDistConcurrency.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerDistConcurrency.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestBlockFixerGeneratedBlockDist.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerGeneratedBlockDist.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestBlockFixerGeneratedBlockDist.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerGeneratedBlockDist.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestBlockFixerParityBlockFixDist.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerParityBlockFixDist.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestBlockFixerParityBlockFixDist.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerParityBlockFixDist.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestDirectoryTraversal.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestDirectoryTraversal.java similarity index 98% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestDirectoryTraversal.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestDirectoryTraversal.java index 93658a5205..4ab1107c02 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestDirectoryTraversal.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestDirectoryTraversal.java @@ -40,7 +40,7 @@ public class TestDirectoryTraversal extends TestCase { final static Log LOG = LogFactory.getLog( "org.apache.hadoop.raid.TestDirectoryTraversal"); final static String TEST_DIR = new File(System.getProperty("test.build.data", - "build/contrib/raid/test/data")).getAbsolutePath(); + "target/test-data")).getAbsolutePath(); MiniDFSCluster dfs = null; FileSystem fs = null; @@ -211,7 +211,7 @@ private void createTestFile(Path file) throws IOException { private void mySetup() throws IOException { conf = new Configuration(); - dfs = new MiniDFSCluster(conf, 6, true, null); + dfs = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); dfs.waitActive(); fs = dfs.getFileSystem(); } diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestErasureCodes.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestErasureCodes.java similarity index 81% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestErasureCodes.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestErasureCodes.java index d1d3f60f9e..6d60b574a3 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestErasureCodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestErasureCodes.java @@ -169,6 +169,57 @@ public void testXorPerformance() { assertTrue("Decode failed", java.util.Arrays.equals(copy, message[0])); } + public void testComputeErrorLocations() { + for (int i = 0; i < TEST_TIMES; ++i) { + verifyErrorLocations(10, 4, 1); + verifyErrorLocations(10, 4, 2); + } + } + + public void verifyErrorLocations(int stripeSize, int paritySize, int errors) { + int[] message = new int[stripeSize]; + int[] parity = new int[paritySize]; + Set errorLocations = new HashSet(); + for (int i = 0; i < message.length; ++i) { + message[i] = RAND.nextInt(256); + } + while (errorLocations.size() < errors) { + int loc = RAND.nextInt(stripeSize + paritySize); + errorLocations.add(loc); + } + ReedSolomonCode codec = new ReedSolomonCode(stripeSize, paritySize); + codec.encode(message, parity); + int[] data = combineArrays(parity, message); + for (Integer i : errorLocations) { + data[i] = randError(data[i]); + } + Set recoveredLocations = new HashSet(); + boolean resolved = codec.computeErrorLocations(data, recoveredLocations); + if (resolved) { + assertEquals(errorLocations, recoveredLocations); + } + } + + private int randError(int actual) { + while (true) { + int r = RAND.nextInt(256); + if (r != actual) { + return r; + } + } + } + + private int[] combineArrays(int[] array1, int[] array2) { + int[] result = new int[array1.length + array2.length]; + for (int i = 0; i < array1.length; ++i) { + result[i] = array1[i]; + } + for (int i = 0; i < array2.length; ++i) { + result[i + array1.length] = array2[i]; + } + return result; + } + private int[] randomErasedLocation(int erasedLen, int dataLen) { int[] erasedLocations = new int[erasedLen]; for (int i = 0; i < erasedLen; i++) { diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestGaloisField.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestGaloisField.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestGaloisField.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestGaloisField.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestHarIndexParser.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestHarIndexParser.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestHarIndexParser.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestHarIndexParser.java diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidFilter.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidFilter.java similarity index 98% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidFilter.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidFilter.java index 4b1fe67205..037c682df3 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidFilter.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidFilter.java @@ -36,7 +36,7 @@ public class TestRaidFilter extends TestCase { final static String TEST_DIR = new File(System.getProperty("test.build.data", - "build/contrib/raid/test/data")).getAbsolutePath(); + "target/test-data")).getAbsolutePath(); final static Log LOG = LogFactory.getLog("org.apache.hadoop.raid.TestRaidFilter"); diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidHar.java similarity index 92% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidHar.java index 34398048b7..d3aeab7bff 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidHar.java @@ -22,6 +22,7 @@ import java.io.FileNotFoundException; import java.util.Random; +import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import junit.framework.TestCase; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -34,6 +35,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MiniMRCluster; /** @@ -41,7 +43,7 @@ */ public class TestRaidHar extends TestCase { final static String TEST_DIR = new File(System.getProperty("test.build.data", - "build/contrib/raid/test/data")).getAbsolutePath(); + "target/test-data")).getAbsolutePath(); final static String CONFIG_FILE = new File(TEST_DIR, "test-raid.xml").getAbsolutePath(); final static long RELOAD_INTERVAL = 1000; @@ -96,11 +98,27 @@ private void createClusters(boolean local) throws Exception { fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(taskTrackers, namenode, 3); - jobTrackerName = "localhost:" + mr.getJobTrackerPort(); + JobConf jobConf = mr.createJobConf(); + jobTrackerName = "localhost:" + jobConf.get(JTConfig.JT_IPC_ADDRESS); hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); FileSystem.setDefaultUri(conf, namenode); conf.set("mapred.job.tracker", jobTrackerName); + conf.set("mapreduce.framework.name", "yarn"); + String rmAdress = jobConf.get("yarn.resourcemanager.address"); + if (rmAdress != null) { + conf.set("yarn.resourcemanager.address", rmAdress); + } + String schedulerAdress = + jobConf.get("yarn.resourcemanager.scheduler.address"); + if (schedulerAdress != null) { + conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress); + } + String jobHistoryAddress = + jobConf.get("mapreduce.jobhistory.address"); + if (jobHistoryAddress != null) { + conf.set("mapreduce.jobhistory.address", jobHistoryAddress); + } } /** diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidNode.java similarity index 88% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidNode.java index 355d9ad79d..5bdee75068 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidNode.java @@ -37,9 +37,13 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.JobContext; import org.apache.hadoop.mapred.MiniMRCluster; +import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.raid.protocol.PolicyInfo; import org.apache.hadoop.raid.protocol.PolicyList; +import org.apache.hadoop.util.JarFinder; import org.apache.hadoop.raid.protocol.PolicyInfo.ErasureCodeType; /** @@ -49,7 +53,8 @@ */ public class TestRaidNode extends TestCase { final static String TEST_DIR = new File(System.getProperty("test.build.data", - "build/contrib/raid/test/data")).getAbsolutePath(); + "target/test-data")).getAbsolutePath(); + public static final String DistRaid_JAR = JarFinder.getJar(DistRaid.class); final static String CONFIG_FILE = new File(TEST_DIR, "test-raid.xml").getAbsolutePath(); final static long RELOAD_INTERVAL = 1000; @@ -76,6 +81,8 @@ private void createClusters(boolean local) throws Exception { conf.setBoolean("raid.config.reload", true); conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL); conf.setBoolean("dfs.permissions.enabled", true); + conf.setLong(JobMonitor.JOBMONITOR_INTERVAL_KEY, 20000); + conf.setLong(RaidNode.TRIGGER_MONITOR_SLEEP_TIME_KEY, 3000L); // scan all policies once every 5 second conf.setLong("raid.policy.rescan.interval", 5000); @@ -103,11 +110,27 @@ private void createClusters(boolean local) throws Exception { namenode = fileSys.getUri().toString(); final int taskTrackers = 4; mr = new MiniMRCluster(taskTrackers, namenode, 3); - jobTrackerName = "localhost:" + mr.getJobTrackerPort(); + JobConf jobConf = mr.createJobConf(); + jobTrackerName = "localhost:" + jobConf.get(JTConfig.JT_IPC_ADDRESS); hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); FileSystem.setDefaultUri(conf, namenode); conf.set("mapred.job.tracker", jobTrackerName); + conf.set("mapreduce.framework.name", "yarn"); + String rmAdress = jobConf.get("yarn.resourcemanager.address"); + if (rmAdress != null) { + conf.set("yarn.resourcemanager.address", rmAdress); + } + String schedulerAdress = + jobConf.get("yarn.resourcemanager.scheduler.address"); + if (schedulerAdress != null) { + conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress); + } + String jobHistoryAddress = + jobConf.get("mapreduce.jobhistory.address"); + if (jobHistoryAddress != null) { + conf.set("mapreduce.jobhistory.address", jobHistoryAddress); + } } class ConfigBuilder { @@ -238,9 +261,9 @@ public void testPathFilter() throws Exception { LOG.info("Test testPathFilter started."); long blockSizes [] = {1024L}; - long stripeLengths [] = {1, 2, 5, 6, 10, 11, 12}; - long targetReplication = 1; - long metaReplication = 1; + int stripeLengths [] = {5, 6, 10, 11, 12}; + int targetReplication = 1; + int metaReplication = 1; int numBlock = 11; int iter = 0; @@ -284,7 +307,8 @@ private void doTestPathFilter(int iter, long targetReplication, LOG.info("doTestPathFilter created test files for iteration " + iter); // create an instance of the RaidNode - cnode = RaidNode.createRaidNode(null, conf); + Configuration localConf = new Configuration(conf); + cnode = RaidNode.createRaidNode(null, localConf); FileStatus[] listPaths = null; // wait till file is raided @@ -314,7 +338,6 @@ private void doTestPathFilter(int iter, long targetReplication, } // assertEquals(listPaths.length, 1); // all files raided LOG.info("doTestPathFilter all files found in Raid."); - Thread.sleep(20000); // Without this wait, unit test crashes // check for error at beginning of file shell = new RaidShell(conf); @@ -466,16 +489,23 @@ private void doCheckPolicy() throws Exception { LOG.info("doCheckPolicy completed:"); } - private void createTestFiles(String path, String destpath) throws IOException { + static public void createTestFiles(FileSystem fileSys, + String path, String destpath, int nfile, + int nblock) throws IOException { + createTestFiles(fileSys, path, destpath, nfile, nblock, (short)1); + } + + static void createTestFiles(FileSystem fileSys, String path, String destpath, int nfile, + int nblock, short repl) throws IOException { long blockSize = 1024L; Path dir = new Path(path); Path destPath = new Path(destpath); fileSys.delete(dir, true); fileSys.delete(destPath, true); - for(int i = 0 ; i < 10; i++){ + for(int i = 0 ; i < nfile; i++){ Path file = new Path(path + "file" + i); - createOldFile(fileSys, file, 1, 7, blockSize); + createOldFile(fileSys, file, repl, nblock, blockSize); } } @@ -499,12 +529,15 @@ public void testDistRaid() throws Exception { RaidNode cnode = null; try { - createTestFiles("/user/dhruba/raidtest/", "/destraid/user/dhruba/raidtest"); - createTestFiles("/user/dhruba/raidtest2/", "/destraid/user/dhruba/raidtest2"); + createTestFiles(fileSys, "/user/dhruba/raidtest/", + "/destraid/user/dhruba/raidtest", 5, 7); + createTestFiles(fileSys, "/user/dhruba/raidtest2/", + "/destraid/user/dhruba/raidtest2", 5, 7); LOG.info("Test testDistRaid created test files"); Configuration localConf = new Configuration(conf); localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); + localConf.set(JobContext.JAR, TestRaidNode.DistRaid_JAR); cnode = RaidNode.createRaidNode(null, localConf); // Verify the policies are parsed correctly for (PolicyList policyList : cnode.getAllPolicies()) { @@ -540,15 +573,13 @@ public void testDistRaid() throws Exception { System.currentTimeMillis() - start < MAX_WAITTIME) { Thread.sleep(1000); } - assertEquals(dcnode.jobMonitor.jobsMonitored(), 2); - + start = System.currentTimeMillis(); while (dcnode.jobMonitor.jobsSucceeded() < 2 && System.currentTimeMillis() - start < MAX_WAITTIME) { Thread.sleep(1000); } - assertEquals(dcnode.jobMonitor.jobsSucceeded(), 2); - + assertEquals(dcnode.jobMonitor.jobsSucceeded(), dcnode.jobMonitor.jobsMonitored()); LOG.info("Test testDistRaid successful."); } catch (Exception e) { @@ -647,24 +678,19 @@ public void testSuspendTraversal() throws Exception { RaidNode cnode = null; try { - createTestFiles( - "/user/dhruba/raidtest/1/", "/destraid/user/dhruba/raidtest/1"); - createTestFiles( - "/user/dhruba/raidtest/2/", "/destraid/user/dhruba/raidtest/2"); - createTestFiles( - "/user/dhruba/raidtest/3/", "/destraid/user/dhruba/raidtest/3"); - createTestFiles( - "/user/dhruba/raidtest/4/", "/destraid/user/dhruba/raidtest/4"); + for(int i = 0; i < 4; i++){ + Path file = new Path("/user/dhruba/raidtest/dir" + i + "/file" + i); + createOldFile(fileSys, file, 1, 7, 1024L); + } + LOG.info("Test testSuspendTraversal created test files"); Configuration localConf = new Configuration(conf); - localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - localConf.setInt("raid.distraid.max.files", 3); + localConf.setInt("raid.distraid.max.jobs", 2); + localConf.setInt("raid.distraid.max.files", 2); localConf.setInt("raid.directorytraversal.threads", 1); - // This is too dependent on the implementation of getFilteredFiles(). - // It relies on the threading behavior where two directories are traversed - // before returning because the list of files is modified in a separate - // thread from the one that decides if there are enough files. + localConf.set(JobContext.JAR, TestRaidNode.DistRaid_JAR); + // 4 test files: 2 jobs with 2 files each. final int numJobsExpected = 2; cnode = RaidNode.createRaidNode(null, localConf); @@ -677,10 +703,20 @@ public void testSuspendTraversal() throws Exception { start = System.currentTimeMillis(); while (dcnode.jobMonitor.jobsSucceeded() < numJobsExpected && System.currentTimeMillis() - start < MAX_WAITTIME) { + LOG.info("Waiting for num jobs succeeded " + dcnode.jobMonitor.jobsSucceeded() + + " to reach " + numJobsExpected); + Thread.sleep(3000); + } + // Wait for any running jobs to finish. + start = System.currentTimeMillis(); + while (dcnode.jobMonitor.runningJobsCount() > 0 && + System.currentTimeMillis() - start < MAX_WAITTIME) { + LOG.info("Waiting for zero running jobs: " + + dcnode.jobMonitor.runningJobsCount()); Thread.sleep(1000); } - assertEquals(dcnode.jobMonitor.jobsMonitored(), numJobsExpected); - assertEquals(dcnode.jobMonitor.jobsSucceeded(), numJobsExpected); + assertEquals(numJobsExpected, dcnode.jobMonitor.jobsMonitored()); + assertEquals(numJobsExpected, dcnode.jobMonitor.jobsSucceeded()); LOG.info("Test testSuspendTraversal successful."); diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidPurge.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidPurge.java similarity index 95% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidPurge.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidPurge.java index accef0b39a..ca6dc4a33d 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidPurge.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidPurge.java @@ -51,6 +51,7 @@ import org.apache.hadoop.raid.protocol.PolicyList; import org.apache.hadoop.hdfs.TestRaidDfs; import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.raid.protocol.PolicyInfo; /** @@ -58,7 +59,7 @@ */ public class TestRaidPurge extends TestCase { final static String TEST_DIR = new File(System.getProperty("test.build.data", - "build/contrib/raid/test/data")).getAbsolutePath(); + "target/test-data")).getAbsolutePath(); final static String CONFIG_FILE = new File(TEST_DIR, "test-raid.xml").getAbsolutePath(); final static long RELOAD_INTERVAL = 1000; @@ -113,11 +114,27 @@ private void createClusters(boolean local) throws Exception { fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(taskTrackers, namenode, 3); - jobTrackerName = "localhost:" + mr.getJobTrackerPort(); + JobConf jobConf = mr.createJobConf(); + jobTrackerName = "localhost:" + jobConf.get(JTConfig.JT_IPC_ADDRESS); hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); FileSystem.setDefaultUri(conf, namenode); conf.set("mapred.job.tracker", jobTrackerName); + conf.set("mapreduce.framework.name", "yarn"); + String rmAdress = jobConf.get("yarn.resourcemanager.address"); + if (rmAdress != null) { + conf.set("yarn.resourcemanager.address", rmAdress); + } + String schedulerAdress = + jobConf.get("yarn.resourcemanager.scheduler.address"); + if (schedulerAdress != null) { + conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress); + } + String jobHistoryAddress = + jobConf.get("mapreduce.jobhistory.address"); + if (jobHistoryAddress != null) { + conf.set("mapreduce.jobhistory.address", jobHistoryAddress); + } } /** @@ -235,6 +252,7 @@ private void doTestPurge(int iter, long targetReplication, // create an instance of the RaidNode Configuration localConf = new Configuration(conf); + localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); cnode = RaidNode.createRaidNode(null, localConf); FileStatus[] listPaths = null; @@ -299,7 +317,7 @@ public void testPurgeHar() throws Exception { createClusters(true); mySetup(1, 1, 5, harDelay); Path dir = new Path("/user/dhruba/raidtest/"); - Path destPath = new Path("/destraid/user/dhruba/raidtest"); + Path destPath = new Path("/raid/user/dhruba/raidtest"); Path file1 = new Path(dir + "/file"); RaidNode cnode = null; try { @@ -308,7 +326,6 @@ public void testPurgeHar() throws Exception { // create an instance of the RaidNode Configuration localConf = new Configuration(conf); - localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); cnode = RaidNode.createRaidNode(null, localConf); // Wait till har is created. @@ -334,14 +351,7 @@ public void testPurgeHar() throws Exception { boolean found = false; FileStatus[] listPaths = null; while (!found || listPaths == null || listPaths.length > 1) { - try { - listPaths = fileSys.listStatus(destPath); - } catch (FileNotFoundException e) { - // If the parent directory is deleted because the har is deleted - // and the parent is empty, try again. - Thread.sleep(1000); - continue; - } + listPaths = fileSys.listStatus(destPath); if (listPaths != null) { for (FileStatus s: listPaths) { LOG.info("testPurgeHar waiting for parity file to be recreated" + diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidShell.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidShell.java similarity index 98% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidShell.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidShell.java index 9375e405e1..c283ce996f 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidShell.java @@ -47,7 +47,7 @@ public class TestRaidShell extends TestCase { final static Log LOG = LogFactory.getLog( "org.apache.hadoop.raid.TestRaidShell"); final static String TEST_DIR = new File(System.getProperty("test.build.data", - "build/contrib/raid/test/data")).getAbsolutePath(); + "target/test-data")).getAbsolutePath(); final static String CONFIG_FILE = new File(TEST_DIR, "test-raid.xml").getAbsolutePath(); final static long RELOAD_INTERVAL = 1000; @@ -249,7 +249,8 @@ private void myTearDown() throws Exception { private long getCRC(FileSystem fs, Path p) throws IOException { CRC32 crc = new CRC32(); FSDataInputStream stm = fs.open(p); - for (int b = 0; b > 0; b = stm.read()) { + int b; + while ((b = stm.read())>=0) { crc.update(b); } stm.close(); diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidShellFsck.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidShellFsck.java similarity index 99% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidShellFsck.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidShellFsck.java index 284162197b..fad14ead74 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidShellFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidShellFsck.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http:www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -51,8 +51,8 @@ public class TestRaidShellFsck { LogFactory.getLog("org.apache.hadoop.raid.TestRaidShellFsck"); final static String TEST_DIR = new File(System. - getProperty("test.build.data", "build/contrib/raid/test/data")). - getAbsolutePath(); + getProperty("test.build.data", "target/test-data")).getAbsolutePath(); + final static String CONFIG_FILE = new File(TEST_DIR, "test-raid.xml"). getAbsolutePath(); final static long RELOAD_INTERVAL = 1000; @@ -262,7 +262,7 @@ private void raidTestFiles(Path raidPath, Path[] filePaths, boolean doHar) } } else { - // case without HAR + // case without HAR for (FileStatus f : listPaths) { Path found = new Path(f.getPath().toUri().getPath()); if (parityFilePath.equals(found)) { diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestReedSolomonDecoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestReedSolomonDecoder.java similarity index 98% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestReedSolomonDecoder.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestReedSolomonDecoder.java index 31704afbe6..5f47cee293 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestReedSolomonDecoder.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestReedSolomonDecoder.java @@ -42,7 +42,7 @@ public class TestReedSolomonDecoder extends TestCase { final static Log LOG = LogFactory.getLog( "org.apache.hadoop.raid.TestReedSolomonDecoder"); final static String TEST_DIR = new File(System.getProperty("test.build.data", - "build/contrib/raid/test/data")).getAbsolutePath(); + "target/test-data")).getAbsolutePath(); final static int NUM_DATANODES = 3; Configuration conf; diff --git a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestReedSolomonEncoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestReedSolomonEncoder.java similarity index 98% rename from hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestReedSolomonEncoder.java rename to hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestReedSolomonEncoder.java index 9815591c54..bd1201413a 100644 --- a/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestReedSolomonEncoder.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestReedSolomonEncoder.java @@ -49,7 +49,7 @@ public class TestReedSolomonEncoder extends TestCase { final static Log LOG = LogFactory.getLog( "org.apache.hadoop.raid.TestReedSolomonEncoder"); final static String TEST_DIR = new File(System.getProperty("test.build.data", - "build/contrib/raid/test/data")).getAbsolutePath(); + "target/test-data")).getAbsolutePath(); final static int NUM_DATANODES = 3; Configuration conf; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index a6df657c8a..1e043f43fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -31,7 +31,7 @@ /** I-node for closed file. */ @InterfaceAudience.Private -class INodeFile extends INode implements BlockCollection { +public class INodeFile extends INode implements BlockCollection { static final FsPermission UMASK = FsPermission.createImmutable((short)0111); //Number of bits for Block size diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml index 27161004a3..38c93fa8bd 100644 --- a/hadoop-hdfs-project/pom.xml +++ b/hadoop-hdfs-project/pom.xml @@ -34,6 +34,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> hadoop-hdfs hadoop-hdfs-httpfs hadoop-hdfs/src/contrib/bkjournal + hadoop-hdfs-raid diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ce0e8e031c..04e09a6a6f 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -118,6 +118,8 @@ Trunk (unreleased changes) MAPREDUCE-3990. MRBench allows Long-sized input-lines value but parses CLI argument as an Integer. (harsh) + MAPREDUCE-3868. Make Raid Compile. (Weiyan Wang via schen) + Branch-2 ( Unreleased changes ) INCOMPATIBLE CHANGES diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 5378c61911..ace8f75f99 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -257,6 +257,12 @@ hadoop-client ${project.version} + + org.apache.hadoop + hadoop-hdfs-raid + ${project.version} + + org.apache.hadoop hadoop-minicluster From f2d9df5580af822c488a3206ef4623d5095c929c Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Tue, 19 Jun 2012 13:06:51 +0000 Subject: [PATCH 58/91] MAPREDUCE-4311. Capacity scheduler.xml does not accept decimal values for capacity and maximum-capacity settings (Karthik Kambatla via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351700 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../CapacitySchedulerConfiguration.java | 28 ++++++++-------- .../scheduler/capacity/ParentQueue.java | 2 +- .../capacity/TestCapacityScheduler.java | 32 +++++++++---------- .../scheduler/capacity/TestLeafQueue.java | 12 +++---- .../scheduler/capacity/TestParentQueue.java | 27 ++++++++++------ .../TestRMWebServicesCapacitySched.java | 4 +-- 7 files changed, 59 insertions(+), 49 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 04e09a6a6f..49bac687ab 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -581,6 +581,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-3927. Shuffle hang when set map.failures.percent (Bhallamudi Venkata Siva Kamesh via tgraves) + MAPREDUCE-4311. Capacity scheduler.xml does not accept decimal values for + capacity and maximum-capacity settings (Karthik Kambatla via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index e13a14d7e5..86e2dd350d 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -80,13 +80,13 @@ public class CapacitySchedulerConfiguration extends Configuration { DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT = 0.1f; @Private - public static final int UNDEFINED = -1; + public static final float UNDEFINED = -1; @Private - public static final int MINIMUM_CAPACITY_VALUE = 1; + public static final float MINIMUM_CAPACITY_VALUE = 1; @Private - public static final int MAXIMUM_CAPACITY_VALUE = 100; + public static final float MAXIMUM_CAPACITY_VALUE = 100; @Private public static final int DEFAULT_USER_LIMIT = 100; @@ -132,8 +132,8 @@ public float getMaximumApplicationMasterResourcePercent() { DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT); } - public int getCapacity(String queue) { - int capacity = getInt(getQueuePrefix(queue) + CAPACITY, UNDEFINED); + public float getCapacity(String queue) { + float capacity = getFloat(getQueuePrefix(queue) + CAPACITY, UNDEFINED); if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) { throw new IllegalArgumentException("Illegal " + "capacity of " + capacity + " for queue " + queue); @@ -143,31 +143,31 @@ public int getCapacity(String queue) { return capacity; } - public void setCapacity(String queue, int capacity) { - setInt(getQueuePrefix(queue) + CAPACITY, capacity); + public void setCapacity(String queue, float capacity) { + setFloat(getQueuePrefix(queue) + CAPACITY, capacity); LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) + ", capacity=" + capacity); } - public int getMaximumCapacity(String queue) { - int maxCapacity = - getInt(getQueuePrefix(queue) + MAXIMUM_CAPACITY, MAXIMUM_CAPACITY_VALUE); + public float getMaximumCapacity(String queue) { + float maxCapacity = getFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY, + MAXIMUM_CAPACITY_VALUE); return maxCapacity; } - public void setMaximumCapacity(String queue, int maxCapacity) { + public void setMaximumCapacity(String queue, float maxCapacity) { if (maxCapacity > MAXIMUM_CAPACITY_VALUE) { throw new IllegalArgumentException("Illegal " + "maximum-capacity of " + maxCapacity + " for queue " + queue); } - setInt(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity); + setFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity); LOG.debug("CSConf - setMaxCapacity: queuePrefix=" + getQueuePrefix(queue) + ", maxCapacity=" + maxCapacity); } public int getUserLimit(String queue) { - int userLimit = - getInt(getQueuePrefix(queue) + USER_LIMIT, DEFAULT_USER_LIMIT); + int userLimit = getInt(getQueuePrefix(queue) + USER_LIMIT, + DEFAULT_USER_LIMIT); return userLimit; } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index cdb1060b44..bd7e988bf0 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -111,7 +111,7 @@ public ParentQueue(CapacitySchedulerContext cs, cs.getConfiguration().getEnableUserMetrics(), cs.getConf()); - int rawCapacity = cs.getConfiguration().getCapacity(getQueuePath()); + float rawCapacity = cs.getConfiguration().getCapacity(getQueuePath()); if (rootQueue && (rawCapacity != CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE)) { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 40d44aa64f..d3ec0357bf 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -59,13 +59,13 @@ public class TestCapacityScheduler { private static final String B1 = B + ".b1"; private static final String B2 = B + ".b2"; private static final String B3 = B + ".b3"; - private static int A_CAPACITY = 10; - private static int B_CAPACITY = 90; - private static int A1_CAPACITY = 30; - private static int A2_CAPACITY = 70; - private static int B1_CAPACITY = 50; - private static int B2_CAPACITY = 30; - private static int B3_CAPACITY = 20; + private static float A_CAPACITY = 10.5f; + private static float B_CAPACITY = 89.5f; + private static float A1_CAPACITY = 30; + private static float A2_CAPACITY = 70; + private static float B1_CAPACITY = 50; + private static float B2_CAPACITY = 30; + private static float B3_CAPACITY = 20; private ResourceManager resourceManager = null; @@ -250,14 +250,14 @@ public void testRefreshQueues() throws Exception { cs.reinitialize(conf, null, null); checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); - conf.setCapacity(A, 80); - conf.setCapacity(B, 20); + conf.setCapacity(A, 80f); + conf.setCapacity(B, 20f); cs.reinitialize(conf, null,null); - checkQueueCapacities(cs, 80, 20); + checkQueueCapacities(cs, 80f, 20f); } private void checkQueueCapacities(CapacityScheduler cs, - int capacityA, int capacityB) { + float capacityA, float capacityB) { CSQueue rootQueue = cs.getRootQueue(); CSQueue queueA = findQueue(rootQueue, A); CSQueue queueB = findQueue(rootQueue, B); @@ -274,13 +274,13 @@ private void checkQueueCapacities(CapacityScheduler cs, checkQueueCapacity(queueB, capB, capB, 1.0f, 1.0f); checkQueueCapacity(queueA1, A1_CAPACITY / 100.0f, (A1_CAPACITY/100.0f) * capA, 1.0f, 1.0f); - checkQueueCapacity(queueA2, (float)A2_CAPACITY / 100.0f, + checkQueueCapacity(queueA2, A2_CAPACITY / 100.0f, (A2_CAPACITY/100.0f) * capA, 1.0f, 1.0f); - checkQueueCapacity(queueB1, (float)B1_CAPACITY / 100.0f, + checkQueueCapacity(queueB1, B1_CAPACITY / 100.0f, (B1_CAPACITY/100.0f) * capB, 1.0f, 1.0f); - checkQueueCapacity(queueB2, (float)B2_CAPACITY / 100.0f, + checkQueueCapacity(queueB2, B2_CAPACITY / 100.0f, (B2_CAPACITY/100.0f) * capB, 1.0f, 1.0f); - checkQueueCapacity(queueB3, (float)B3_CAPACITY / 100.0f, + checkQueueCapacity(queueB3, B3_CAPACITY / 100.0f, (B3_CAPACITY/100.0f) * capB, 1.0f, 1.0f); } @@ -340,7 +340,7 @@ public void testParseQueue() throws IOException { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} ); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100); + conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); cs.reinitialize(conf, null, null); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 7571a0fc86..f74dab4459 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -135,7 +135,7 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; - conf.setCapacity(Q_A, 9); + conf.setCapacity(Q_A, 8.5f); conf.setMaximumCapacity(Q_A, 20); conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*"); @@ -145,7 +145,7 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*"); final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C; - conf.setCapacity(Q_C, 1); + conf.setCapacity(Q_C, 1.5f); conf.setMaximumCapacity(Q_C, 10); conf.setAcl(Q_C, QueueACL.SUBMIT_APPLICATIONS, " "); @@ -208,8 +208,8 @@ public void testInitializeQueue() throws Exception { //can add more sturdy test with 3-layer queues //once MAPREDUCE:3410 is resolved LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); - assertEquals(0.09, a.getCapacity(), epsilon); - assertEquals(0.09, a.getAbsoluteCapacity(), epsilon); + assertEquals(0.085, a.getCapacity(), epsilon); + assertEquals(0.085, a.getAbsoluteCapacity(), epsilon); assertEquals(0.2, a.getMaximumCapacity(), epsilon); assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon); @@ -220,8 +220,8 @@ public void testInitializeQueue() throws Exception { assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon); ParentQueue c = (ParentQueue)queues.get(C); - assertEquals(0.01, c.getCapacity(), epsilon); - assertEquals(0.01, c.getAbsoluteCapacity(), epsilon); + assertEquals(0.015, c.getCapacity(), epsilon); + assertEquals(0.015, c.getAbsoluteCapacity(), epsilon); assertEquals(0.1, c.getMaximumCapacity(), epsilon); assertEquals(0.1, c.getAbsoluteMaximumCapacity(), epsilon); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 9e80b796a7..c4c48497d3 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -18,16 +18,27 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.when; import java.util.HashMap; -import java.util.Map; - import java.util.List; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -35,10 +46,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.QueueACL; -import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; - import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -289,10 +296,10 @@ private void setupMultiLevelQueues(CapacitySchedulerConfiguration conf) { conf.setCapacity(Q_B, 50); final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C; - conf.setCapacity(Q_C, 20); + conf.setCapacity(Q_C, 19.5f); final String Q_D = CapacitySchedulerConfiguration.ROOT + "." + D; - conf.setCapacity(Q_D, 20); + conf.setCapacity(Q_D, 20.5f); // Define 2-nd level queues conf.setQueues(Q_A, new String[] {A1, A2}); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index b8dc072ba3..c0b58619cd 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -124,11 +124,11 @@ private static void setupQueueConfiguration( conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); final String A = CapacitySchedulerConfiguration.ROOT + ".a"; - conf.setCapacity(A, 10); + conf.setCapacity(A, 10.5f); conf.setMaximumCapacity(A, 50); final String B = CapacitySchedulerConfiguration.ROOT + ".b"; - conf.setCapacity(B, 90); + conf.setCapacity(B, 89.5f); // Define 2nd-level queues final String A1 = A + ".a1"; From cfa26ceff3fabceb8a9c1b199e098af07d9886b6 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Tue, 19 Jun 2012 16:17:21 +0000 Subject: [PATCH 59/91] MAPREDUCE-4341. add types to capacity scheduler properties documentation (Karthik Kambatla via tgraves). git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351762 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 49bac687ab..206f9a17c5 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -584,6 +584,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4311. Capacity scheduler.xml does not accept decimal values for capacity and maximum-capacity settings (Karthik Kambatla via tgraves) + MAPREDUCE-4341. add types to capacity scheduler properties documentation + (Karthik Kambatla via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm index 0ee8220425..01a9f60376 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm @@ -197,14 +197,14 @@ Hadoop MapReduce Next Generation - Capacity Scheduler || Property || Description | *--------------------------------------+--------------------------------------+ | <<.capacity>>> | | -| | Queue in percentage (%). | +| | Queue in percentage (%) as a float (e.g. 12.5).| | | The sum of capacities for all queues, at each level, must be equal | | | to 100. | | | Applications in the queue may consume more resources than the queue's | | | capacity if there are free resources, providing elasticity. | *--------------------------------------+--------------------------------------+ | <<.maximum-capacity>>> | | -| | Maximum queue capacity in percentage (%). | +| | Maximum queue capacity in percentage (%) as a float. | | | This limits the for applications in the queue. | | | Defaults to -1 which disables it. | *--------------------------------------+--------------------------------------+ From 92c1005d35c3618ce3894e471df1f1751c7d7c34 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Tue, 19 Jun 2012 19:40:44 +0000 Subject: [PATCH 60/91] MAPREDUCE-4267. mavenize pipes (tgraves via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351818 13f79535-47bb-0310-9956-ffa450edef68 --- .../resources/assemblies/hadoop-tools.xml | 67 +++++++++++ hadoop-mapreduce-project/CHANGES.txt | 2 + .../src/c++/pipes/.autom4te.cfg | 42 ------- .../src/c++/pipes/Makefile.am | 31 ----- .../src/c++/pipes/configure.ac | 57 --------- .../src/c++/utils/.autom4te.cfg | 42 ------- .../src/c++/utils/Makefile.am | 33 ------ .../src/c++/utils/configure.ac | 56 --------- .../src/c++/utils/m4/hadoop_utils.m4 | 68 ----------- .../src/examples/pipes/.autom4te.cfg | 42 ------- .../src/examples/pipes/Makefile.am | 36 ------ .../src/examples/pipes/configure.ac | 58 ---------- hadoop-tools/hadoop-pipes/pom.xml | 108 ++++++++++++++++++ hadoop-tools/hadoop-pipes/src/CMakeLists.txt | 99 ++++++++++++++++ .../src/main/native/examples}/README.txt | 8 +- .../main/native/examples}/conf/word-part.xml | 0 .../src/main/native/examples}/conf/word.xml | 0 .../src/main/native/examples}/impl/sort.cc | 0 .../native/examples}/impl/wordcount-nopipe.cc | 0 .../native/examples}/impl/wordcount-part.cc | 0 .../native/examples}/impl/wordcount-simple.cc | 0 .../main/native}/pipes/api/hadoop/Pipes.hh | 0 .../pipes/api/hadoop/TemplateFactory.hh | 0 .../debug/pipes-default-gdb-commands.txt | 0 .../native}/pipes/debug/pipes-default-script | 0 .../main/native}/pipes/impl/HadoopPipes.cc | 0 .../native}/utils/api/hadoop/SerialUtils.hh | 0 .../native}/utils/api/hadoop/StringUtils.hh | 0 .../main/native}/utils/impl/SerialUtils.cc | 0 .../main/native}/utils/impl/StringUtils.cc | 0 hadoop-tools/hadoop-tools-dist/pom.xml | 47 ++++++++ hadoop-tools/pom.xml | 1 + 32 files changed, 328 insertions(+), 469 deletions(-) create mode 100644 hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml delete mode 100644 hadoop-mapreduce-project/src/c++/pipes/.autom4te.cfg delete mode 100644 hadoop-mapreduce-project/src/c++/pipes/Makefile.am delete mode 100644 hadoop-mapreduce-project/src/c++/pipes/configure.ac delete mode 100644 hadoop-mapreduce-project/src/c++/utils/.autom4te.cfg delete mode 100644 hadoop-mapreduce-project/src/c++/utils/Makefile.am delete mode 100644 hadoop-mapreduce-project/src/c++/utils/configure.ac delete mode 100644 hadoop-mapreduce-project/src/c++/utils/m4/hadoop_utils.m4 delete mode 100644 hadoop-mapreduce-project/src/examples/pipes/.autom4te.cfg delete mode 100644 hadoop-mapreduce-project/src/examples/pipes/Makefile.am delete mode 100644 hadoop-mapreduce-project/src/examples/pipes/configure.ac create mode 100644 hadoop-tools/hadoop-pipes/pom.xml create mode 100644 hadoop-tools/hadoop-pipes/src/CMakeLists.txt rename {hadoop-mapreduce-project/src/examples/pipes => hadoop-tools/hadoop-pipes/src/main/native/examples}/README.txt (50%) rename {hadoop-mapreduce-project/src/examples/pipes => hadoop-tools/hadoop-pipes/src/main/native/examples}/conf/word-part.xml (100%) rename {hadoop-mapreduce-project/src/examples/pipes => hadoop-tools/hadoop-pipes/src/main/native/examples}/conf/word.xml (100%) rename {hadoop-mapreduce-project/src/examples/pipes => hadoop-tools/hadoop-pipes/src/main/native/examples}/impl/sort.cc (100%) rename {hadoop-mapreduce-project/src/examples/pipes => hadoop-tools/hadoop-pipes/src/main/native/examples}/impl/wordcount-nopipe.cc (100%) rename {hadoop-mapreduce-project/src/examples/pipes => hadoop-tools/hadoop-pipes/src/main/native/examples}/impl/wordcount-part.cc (100%) rename {hadoop-mapreduce-project/src/examples/pipes => hadoop-tools/hadoop-pipes/src/main/native/examples}/impl/wordcount-simple.cc (100%) rename {hadoop-mapreduce-project/src/c++ => hadoop-tools/hadoop-pipes/src/main/native}/pipes/api/hadoop/Pipes.hh (100%) rename {hadoop-mapreduce-project/src/c++ => hadoop-tools/hadoop-pipes/src/main/native}/pipes/api/hadoop/TemplateFactory.hh (100%) rename {hadoop-mapreduce-project/src/c++ => hadoop-tools/hadoop-pipes/src/main/native}/pipes/debug/pipes-default-gdb-commands.txt (100%) rename {hadoop-mapreduce-project/src/c++ => hadoop-tools/hadoop-pipes/src/main/native}/pipes/debug/pipes-default-script (100%) rename {hadoop-mapreduce-project/src/c++ => hadoop-tools/hadoop-pipes/src/main/native}/pipes/impl/HadoopPipes.cc (100%) rename {hadoop-mapreduce-project/src/c++ => hadoop-tools/hadoop-pipes/src/main/native}/utils/api/hadoop/SerialUtils.hh (100%) rename {hadoop-mapreduce-project/src/c++ => hadoop-tools/hadoop-pipes/src/main/native}/utils/api/hadoop/StringUtils.hh (100%) rename {hadoop-mapreduce-project/src/c++ => hadoop-tools/hadoop-pipes/src/main/native}/utils/impl/SerialUtils.cc (100%) rename {hadoop-mapreduce-project/src/c++ => hadoop-tools/hadoop-pipes/src/main/native}/utils/impl/StringUtils.cc (100%) diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml new file mode 100644 index 0000000000..1e3356d337 --- /dev/null +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml @@ -0,0 +1,67 @@ + + + hadoop-tools + + dir + + false + + + ../hadoop-pipes/src/main/native/pipes/api/hadoop + + *.hh + + /include + + + ../hadoop-pipes/src/main/native/utils/api/hadoop + + *.hh + + /include + + + ../hadoop-pipes/target/native + + *.a + + lib/native + + + + + /share/hadoop/${hadoop.component}/lib + false + runtime + false + + + org.apache.hadoop:hadoop-common + org.apache.hadoop:hadoop-hdfs + org.apache.hadoop:hadoop-mapreduce + + org.apache.hadoop:hadoop-pipes + + org.slf4j:slf4j-api + org.slf4j:slf4j-log4j12 + + + + diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 206f9a17c5..29da54d5db 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -394,6 +394,8 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4301. Dedupe some strings in MRAM for memory savings (bobby via tgraves) + MAPREDUCE-4267. mavenize pipes (tgraves via bobby) + OPTIMIZATIONS MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn diff --git a/hadoop-mapreduce-project/src/c++/pipes/.autom4te.cfg b/hadoop-mapreduce-project/src/c++/pipes/.autom4te.cfg deleted file mode 100644 index d21d1c9877..0000000000 --- a/hadoop-mapreduce-project/src/c++/pipes/.autom4te.cfg +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# autom4te configuration for hadoop utils library -# - -begin-language: "Autoheader-preselections" -args: --no-cache -end-language: "Autoheader-preselections" - -begin-language: "Automake-preselections" -args: --no-cache -end-language: "Automake-preselections" - -begin-language: "Autoreconf-preselections" -args: --no-cache -end-language: "Autoreconf-preselections" - -begin-language: "Autoconf-without-aclocal-m4" -args: --no-cache -end-language: "Autoconf-without-aclocal-m4" - -begin-language: "Autoconf" -args: --no-cache -end-language: "Autoconf" - diff --git a/hadoop-mapreduce-project/src/c++/pipes/Makefile.am b/hadoop-mapreduce-project/src/c++/pipes/Makefile.am deleted file mode 100644 index 2c91d7f809..0000000000 --- a/hadoop-mapreduce-project/src/c++/pipes/Makefile.am +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ACLOCAL_AMFLAGS = -I ../utils/m4 -AM_CXXFLAGS=-I$(srcdir)/api -Wall -I$(HADOOP_UTILS_PREFIX)/include - -# List the api header files and where they will be installed -apidir = $(includedir)/hadoop -api_HEADERS = \ - api/hadoop/Pipes.hh \ - api/hadoop/TemplateFactory.hh - -# Define the libaries that need to be built -lib_LIBRARIES = libhadooppipes.a - -# Define the sources for lib -libhadooppipes_a_SOURCES = \ - impl/HadoopPipes.cc - diff --git a/hadoop-mapreduce-project/src/c++/pipes/configure.ac b/hadoop-mapreduce-project/src/c++/pipes/configure.ac deleted file mode 100644 index ac05ece0f5..0000000000 --- a/hadoop-mapreduce-project/src/c++/pipes/configure.ac +++ /dev/null @@ -1,57 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -*- Autoconf -*- -# Process this file with autoconf to produce a configure script. - -AC_PREREQ(2.59) -AC_INIT(hadoop-pipes, 0.13.0, omalley@apache.org) - -AC_CONFIG_AUX_DIR([config]) -AC_CONFIG_MACRO_DIR([../utils/m4]) - -AM_INIT_AUTOMAKE([subdir-objects foreign no-dist]) - -AC_CONFIG_SRCDIR([impl/HadoopPipes.cc]) -AC_CONFIG_HEADER([impl/config.h]) -AC_CONFIG_FILES([Makefile]) - -AC_PREFIX_DEFAULT(`pwd`/../install) - -USE_HADOOP_UTILS -HADOOP_PIPES_SETUP -CHECK_INSTALL_CFLAG - -# Checks for programs. -AC_PROG_CXX -AC_PROG_LIBTOOL - -# Checks for libraries. - -# Checks for header files. -AC_LANG(C++) -AC_CHECK_HEADERS([unistd.h]) - -# Checks for typedefs, structures, and compiler characteristics. -AC_HEADER_STDBOOL -AC_C_CONST -AC_TYPE_OFF_T -AC_TYPE_SIZE_T -AC_FUNC_STRERROR_R - -# Checks for library functions. -AC_CHECK_FUNCS([mkdir uname]) -AC_OUTPUT diff --git a/hadoop-mapreduce-project/src/c++/utils/.autom4te.cfg b/hadoop-mapreduce-project/src/c++/utils/.autom4te.cfg deleted file mode 100644 index d21d1c9877..0000000000 --- a/hadoop-mapreduce-project/src/c++/utils/.autom4te.cfg +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# autom4te configuration for hadoop utils library -# - -begin-language: "Autoheader-preselections" -args: --no-cache -end-language: "Autoheader-preselections" - -begin-language: "Automake-preselections" -args: --no-cache -end-language: "Automake-preselections" - -begin-language: "Autoreconf-preselections" -args: --no-cache -end-language: "Autoreconf-preselections" - -begin-language: "Autoconf-without-aclocal-m4" -args: --no-cache -end-language: "Autoconf-without-aclocal-m4" - -begin-language: "Autoconf" -args: --no-cache -end-language: "Autoconf" - diff --git a/hadoop-mapreduce-project/src/c++/utils/Makefile.am b/hadoop-mapreduce-project/src/c++/utils/Makefile.am deleted file mode 100644 index d99ea14958..0000000000 --- a/hadoop-mapreduce-project/src/c++/utils/Makefile.am +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ACLOCAL_AMFLAGS = -I m4 -AM_CXXFLAGS=-I$(srcdir)/api -Wall - -# List the api header files and where they will be installed -apidir = $(includedir)/hadoop -api_HEADERS = \ - api/hadoop/StringUtils.hh \ - api/hadoop/SerialUtils.hh - - -# Define the libaries that need to be built -lib_LIBRARIES = libhadooputils.a - -# Define the sources for lib -libhadooputils_a_SOURCES = \ - impl/StringUtils.cc \ - impl/SerialUtils.cc - diff --git a/hadoop-mapreduce-project/src/c++/utils/configure.ac b/hadoop-mapreduce-project/src/c++/utils/configure.ac deleted file mode 100644 index 92831816d5..0000000000 --- a/hadoop-mapreduce-project/src/c++/utils/configure.ac +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -*- Autoconf -*- -# Process this file with autoconf to produce a configure script. - -AC_PREREQ(2.59) -AC_INIT(hadoop-utils, 0.13.0, omalley@apache.org) - -AC_CONFIG_AUX_DIR([config]) -AC_CONFIG_MACRO_DIR([m4]) - -AM_INIT_AUTOMAKE([subdir-objects foreign no-dist]) - -AC_CONFIG_SRCDIR([impl/SerialUtils.cc]) -AC_CONFIG_HEADER([impl/config.h]) -AC_CONFIG_FILES([Makefile]) - -AC_PREFIX_DEFAULT(`pwd`/../install) - -CHECK_INSTALL_CFLAG -HADOOP_UTILS_SETUP - -# Checks for programs. -AC_PROG_CXX -AC_PROG_LIBTOOL - -# Checks for libraries. - -# Checks for header files. -AC_LANG(C++) -AC_CHECK_HEADERS([unistd.h]) - -# Checks for typedefs, structures, and compiler characteristics. -AC_HEADER_STDBOOL -AC_C_CONST -AC_TYPE_OFF_T -AC_TYPE_SIZE_T -AC_FUNC_STRERROR_R - -# Checks for library functions. -AC_CHECK_FUNCS([mkdir uname]) -AC_OUTPUT diff --git a/hadoop-mapreduce-project/src/c++/utils/m4/hadoop_utils.m4 b/hadoop-mapreduce-project/src/c++/utils/m4/hadoop_utils.m4 deleted file mode 100644 index d0ed6c4a0d..0000000000 --- a/hadoop-mapreduce-project/src/c++/utils/m4/hadoop_utils.m4 +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# hadoop_utils.m4 - -# Check to see if the install program supports -C -# If so, use "install -C" for the headers. Otherwise, every install -# updates the timestamps on the installed headers, which causes a recompilation -# of any downstream libraries. -AC_DEFUN([CHECK_INSTALL_CFLAG],[ -AC_REQUIRE([AC_PROG_INSTALL]) -touch foo -if $INSTALL -C foo bar; then - INSTALL_DATA="$INSTALL_DATA -C" -fi -rm -f foo bar -]) - -# Set up the things we need for compiling hadoop utils -AC_DEFUN([HADOOP_UTILS_SETUP],[ -AC_REQUIRE([AC_GNU_SOURCE]) -AC_REQUIRE([AC_SYS_LARGEFILE]) -]) - -# define a macro for using hadoop utils -AC_DEFUN([USE_HADOOP_UTILS],[ -AC_REQUIRE([HADOOP_UTILS_SETUP]) -AC_ARG_WITH([hadoop-utils], - AS_HELP_STRING([--with-hadoop-utils=

], - [directory to get hadoop_utils from]), - [HADOOP_UTILS_PREFIX="$withval"], - [HADOOP_UTILS_PREFIX="\${prefix}"]) -AC_SUBST(HADOOP_UTILS_PREFIX) -]) - -AC_DEFUN([HADOOP_PIPES_SETUP],[ -AC_CHECK_HEADERS([pthread.h], [], - AC_MSG_ERROR(Please check if you have installed the pthread library)) -AC_CHECK_LIB([pthread], [pthread_create], [], - AC_MSG_ERROR(Cannot find libpthread.so, please check)) -AC_CHECK_LIB([crypto], [HMAC_Init], [], - AC_MSG_ERROR(Cannot find libcrypto.so, please check)) -]) - -# define a macro for using hadoop pipes -AC_DEFUN([USE_HADOOP_PIPES],[ -AC_REQUIRE([USE_HADOOP_UTILS]) -AC_REQUIRE([HADOOP_PIPES_SETUP]) -AC_ARG_WITH([hadoop-pipes], - AS_HELP_STRING([--with-hadoop-pipes=], - [directory to get hadoop pipes from]), - [HADOOP_PIPES_PREFIX="$withval"], - [HADOOP_PIPES_PREFIX="\${prefix}"]) -AC_SUBST(HADOOP_PIPES_PREFIX) -]) diff --git a/hadoop-mapreduce-project/src/examples/pipes/.autom4te.cfg b/hadoop-mapreduce-project/src/examples/pipes/.autom4te.cfg deleted file mode 100644 index d21d1c9877..0000000000 --- a/hadoop-mapreduce-project/src/examples/pipes/.autom4te.cfg +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# autom4te configuration for hadoop utils library -# - -begin-language: "Autoheader-preselections" -args: --no-cache -end-language: "Autoheader-preselections" - -begin-language: "Automake-preselections" -args: --no-cache -end-language: "Automake-preselections" - -begin-language: "Autoreconf-preselections" -args: --no-cache -end-language: "Autoreconf-preselections" - -begin-language: "Autoconf-without-aclocal-m4" -args: --no-cache -end-language: "Autoconf-without-aclocal-m4" - -begin-language: "Autoconf" -args: --no-cache -end-language: "Autoconf" - diff --git a/hadoop-mapreduce-project/src/examples/pipes/Makefile.am b/hadoop-mapreduce-project/src/examples/pipes/Makefile.am deleted file mode 100644 index 731ab1e4b3..0000000000 --- a/hadoop-mapreduce-project/src/examples/pipes/Makefile.am +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ACLOCAL_AMFLAGS = -I ../../c++/utils/m4 -AM_CXXFLAGS=-Wall -I$(HADOOP_UTILS_PREFIX)/include \ - -I$(HADOOP_PIPES_PREFIX)/include -LDADD=-L$(HADOOP_UTILS_PREFIX)/lib -L$(HADOOP_PIPES_PREFIX)/lib \ - -lhadooppipes -lhadooputils - -bin_PROGRAMS= wordcount-simple wordcount-part wordcount-nopipe pipes-sort - -# Define the sources for each program -wordcount_simple_SOURCES = \ - impl/wordcount-simple.cc - -wordcount_part_SOURCES = \ - impl/wordcount-part.cc - -wordcount_nopipe_SOURCES = \ - impl/wordcount-nopipe.cc - -pipes_sort_SOURCES = \ - impl/sort.cc - diff --git a/hadoop-mapreduce-project/src/examples/pipes/configure.ac b/hadoop-mapreduce-project/src/examples/pipes/configure.ac deleted file mode 100644 index 7959ba64f0..0000000000 --- a/hadoop-mapreduce-project/src/examples/pipes/configure.ac +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -*- Autoconf -*- -# Process this file with autoconf to produce a configure script. - -AC_PREREQ(2.59) -AC_INIT(hadoop-pipes-examples, 0.13.0, omalley@apache.org) - -AC_CONFIG_AUX_DIR([config]) -AC_CONFIG_MACRO_DIR([../../c++/utils/m4]) - -AM_INIT_AUTOMAKE([subdir-objects foreign no-dist]) - -AC_CONFIG_SRCDIR([impl/wordcount-simple.cc]) -AC_CONFIG_HEADER([impl/config.h]) -AC_CONFIG_FILES([Makefile]) - -AC_PREFIX_DEFAULT(`pwd`/../install) - -USE_HADOOP_PIPES - -# Checks for programs. -AC_PROG_CXX -AC_PROG_INSTALL -AC_PROG_LIBTOOL - -# Checks for libraries. - -# Checks for header files. -AC_LANG(C++) -AC_CHECK_HEADERS([unistd.h]) - -# Checks for typedefs, structures, and compiler characteristics. -AC_HEADER_STDBOOL -AC_C_CONST -AC_TYPE_OFF_T -AC_TYPE_SIZE_T -AC_FUNC_STRERROR_R - -# Checks for library functions. -AC_CHECK_FUNCS([mkdir uname]) -AC_CHECK_LIB([socket],[shutdown]) -AC_CHECK_LIB([nsl],[xdr_float]) -AC_OUTPUT diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml new file mode 100644 index 0000000000..70875f26c4 --- /dev/null +++ b/hadoop-tools/hadoop-pipes/pom.xml @@ -0,0 +1,108 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.0.0-SNAPSHOT + ../../hadoop-project + + org.apache.hadoop + hadoop-pipes + 3.0.0-SNAPSHOT + Apache Hadoop Pipes + Apache Hadoop Pipes + pom + + + ${project.build.directory}/log + + + + + native + + false + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + make + compile + run + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-tools/hadoop-pipes/src/CMakeLists.txt b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt new file mode 100644 index 0000000000..8ab7d27812 --- /dev/null +++ b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt @@ -0,0 +1,99 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cmake_minimum_required(VERSION 2.6 FATAL_ERROR) +find_package(OpenSSL REQUIRED) + +set(CMAKE_BUILD_TYPE, Release) + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -O2") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64") + +if (JVM_ARCH_DATA_MODEL EQUAL 32) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32") + set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32") + if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") + set(CMAKE_SYSTEM_PROCESSOR "i686") + endif () +endif (JVM_ARCH_DATA_MODEL EQUAL 32) + +function(output_directory TGT DIR) + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") +endfunction(output_directory TGT DIR) + +include_directories( + main/native/utils/api + main/native/pipes/api + ${CMAKE_CURRENT_SOURCE_DIR} + ${OPENSSL_INCLUDE_DIR} +) + +# Example programs +add_executable(wordcount-simple main/native/examples/impl/wordcount-simple.cc) +target_link_libraries(wordcount-simple hadooppipes hadooputils) + +add_executable(wordcount-part main/native/examples/impl/wordcount-part.cc) +target_link_libraries(wordcount-part hadooppipes hadooputils) + +add_executable(wordcount-nopipe main/native/examples/impl/wordcount-nopipe.cc) +target_link_libraries(wordcount-nopipe hadooppipes hadooputils) + +add_executable(pipes-sort main/native/examples/impl/sort.cc) +target_link_libraries(pipes-sort hadooppipes hadooputils) + +install(TARGETS wordcount-simple wordcount-part wordcount-nopipe pipes-sort + RUNTIME DESTINATION bin +) + +add_library(hadooputils STATIC + main/native/utils/impl/StringUtils.cc + main/native/utils/impl/SerialUtils.cc +) + +install(FILES + main/native/utils/api/hadoop/SerialUtils.hh + main/native/utils/api/hadoop/StringUtils.hh + DESTINATION api/hadoop + COMPONENT headers +) +install(TARGETS hadooputils DESTINATION lib) + +add_library(hadooppipes STATIC + main/native/pipes/impl/HadoopPipes.cc +) +target_link_libraries(hadooppipes + ${JAVA_JVM_LIBRARY} + ${OPENSSL_LIBRARIES} + pthread +) + +install(FILES + main/native/pipes/api/hadoop/Pipes.hh + main/native/pipes/api/hadoop/TemplateFactory.hh + DESTINATION api/hadoop + COMPONENT headers +) +install(TARGETS hadooppipes DESTINATION lib) diff --git a/hadoop-mapreduce-project/src/examples/pipes/README.txt b/hadoop-tools/hadoop-pipes/src/main/native/examples/README.txt similarity index 50% rename from hadoop-mapreduce-project/src/examples/pipes/README.txt rename to hadoop-tools/hadoop-pipes/src/main/native/examples/README.txt index 4685304332..b9448f558a 100644 --- a/hadoop-mapreduce-project/src/examples/pipes/README.txt +++ b/hadoop-tools/hadoop-pipes/src/main/native/examples/README.txt @@ -1,16 +1,16 @@ To run the examples, first compile them: -% ant -Dcompile.c++=yes examples +% mvn install and then copy the binaries to dfs: -% bin/hadoop fs -put build/c++-examples/Linux-i386-32/bin /examples/bin +% hadoop fs -put target/native/wordcount-simple /examples/bin/ create an input directory with text files: -% bin/hadoop fs -put my-data in-dir +% hadoop fs -put my-data in-dir and run the word count example: -% bin/hadoop pipes -conf src/examples/pipes/conf/word.xml \ +% hadoop pipes -conf src/main/native/examples/conf/word.xml \ -input in-dir -output out-dir diff --git a/hadoop-mapreduce-project/src/examples/pipes/conf/word-part.xml b/hadoop-tools/hadoop-pipes/src/main/native/examples/conf/word-part.xml similarity index 100% rename from hadoop-mapreduce-project/src/examples/pipes/conf/word-part.xml rename to hadoop-tools/hadoop-pipes/src/main/native/examples/conf/word-part.xml diff --git a/hadoop-mapreduce-project/src/examples/pipes/conf/word.xml b/hadoop-tools/hadoop-pipes/src/main/native/examples/conf/word.xml similarity index 100% rename from hadoop-mapreduce-project/src/examples/pipes/conf/word.xml rename to hadoop-tools/hadoop-pipes/src/main/native/examples/conf/word.xml diff --git a/hadoop-mapreduce-project/src/examples/pipes/impl/sort.cc b/hadoop-tools/hadoop-pipes/src/main/native/examples/impl/sort.cc similarity index 100% rename from hadoop-mapreduce-project/src/examples/pipes/impl/sort.cc rename to hadoop-tools/hadoop-pipes/src/main/native/examples/impl/sort.cc diff --git a/hadoop-mapreduce-project/src/examples/pipes/impl/wordcount-nopipe.cc b/hadoop-tools/hadoop-pipes/src/main/native/examples/impl/wordcount-nopipe.cc similarity index 100% rename from hadoop-mapreduce-project/src/examples/pipes/impl/wordcount-nopipe.cc rename to hadoop-tools/hadoop-pipes/src/main/native/examples/impl/wordcount-nopipe.cc diff --git a/hadoop-mapreduce-project/src/examples/pipes/impl/wordcount-part.cc b/hadoop-tools/hadoop-pipes/src/main/native/examples/impl/wordcount-part.cc similarity index 100% rename from hadoop-mapreduce-project/src/examples/pipes/impl/wordcount-part.cc rename to hadoop-tools/hadoop-pipes/src/main/native/examples/impl/wordcount-part.cc diff --git a/hadoop-mapreduce-project/src/examples/pipes/impl/wordcount-simple.cc b/hadoop-tools/hadoop-pipes/src/main/native/examples/impl/wordcount-simple.cc similarity index 100% rename from hadoop-mapreduce-project/src/examples/pipes/impl/wordcount-simple.cc rename to hadoop-tools/hadoop-pipes/src/main/native/examples/impl/wordcount-simple.cc diff --git a/hadoop-mapreduce-project/src/c++/pipes/api/hadoop/Pipes.hh b/hadoop-tools/hadoop-pipes/src/main/native/pipes/api/hadoop/Pipes.hh similarity index 100% rename from hadoop-mapreduce-project/src/c++/pipes/api/hadoop/Pipes.hh rename to hadoop-tools/hadoop-pipes/src/main/native/pipes/api/hadoop/Pipes.hh diff --git a/hadoop-mapreduce-project/src/c++/pipes/api/hadoop/TemplateFactory.hh b/hadoop-tools/hadoop-pipes/src/main/native/pipes/api/hadoop/TemplateFactory.hh similarity index 100% rename from hadoop-mapreduce-project/src/c++/pipes/api/hadoop/TemplateFactory.hh rename to hadoop-tools/hadoop-pipes/src/main/native/pipes/api/hadoop/TemplateFactory.hh diff --git a/hadoop-mapreduce-project/src/c++/pipes/debug/pipes-default-gdb-commands.txt b/hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-gdb-commands.txt similarity index 100% rename from hadoop-mapreduce-project/src/c++/pipes/debug/pipes-default-gdb-commands.txt rename to hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-gdb-commands.txt diff --git a/hadoop-mapreduce-project/src/c++/pipes/debug/pipes-default-script b/hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-script similarity index 100% rename from hadoop-mapreduce-project/src/c++/pipes/debug/pipes-default-script rename to hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-script diff --git a/hadoop-mapreduce-project/src/c++/pipes/impl/HadoopPipes.cc b/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc similarity index 100% rename from hadoop-mapreduce-project/src/c++/pipes/impl/HadoopPipes.cc rename to hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc diff --git a/hadoop-mapreduce-project/src/c++/utils/api/hadoop/SerialUtils.hh b/hadoop-tools/hadoop-pipes/src/main/native/utils/api/hadoop/SerialUtils.hh similarity index 100% rename from hadoop-mapreduce-project/src/c++/utils/api/hadoop/SerialUtils.hh rename to hadoop-tools/hadoop-pipes/src/main/native/utils/api/hadoop/SerialUtils.hh diff --git a/hadoop-mapreduce-project/src/c++/utils/api/hadoop/StringUtils.hh b/hadoop-tools/hadoop-pipes/src/main/native/utils/api/hadoop/StringUtils.hh similarity index 100% rename from hadoop-mapreduce-project/src/c++/utils/api/hadoop/StringUtils.hh rename to hadoop-tools/hadoop-pipes/src/main/native/utils/api/hadoop/StringUtils.hh diff --git a/hadoop-mapreduce-project/src/c++/utils/impl/SerialUtils.cc b/hadoop-tools/hadoop-pipes/src/main/native/utils/impl/SerialUtils.cc similarity index 100% rename from hadoop-mapreduce-project/src/c++/utils/impl/SerialUtils.cc rename to hadoop-tools/hadoop-pipes/src/main/native/utils/impl/SerialUtils.cc diff --git a/hadoop-mapreduce-project/src/c++/utils/impl/StringUtils.cc b/hadoop-tools/hadoop-pipes/src/main/native/utils/impl/StringUtils.cc similarity index 100% rename from hadoop-mapreduce-project/src/c++/utils/impl/StringUtils.cc rename to hadoop-tools/hadoop-pipes/src/main/native/utils/impl/StringUtils.cc diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml index 330cfd3731..951eab139c 100644 --- a/hadoop-tools/hadoop-tools-dist/pom.xml +++ b/hadoop-tools/hadoop-tools-dist/pom.xml @@ -70,6 +70,13 @@ hadoop-gridmix compile + + org.apache.hadoop + hadoop-pipes + compile + pom + ${project.version} + @@ -92,4 +99,44 @@ + + + dist + + false + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.hadoop + hadoop-assemblies + ${project.version} + + + + + dist + prepare-package + + single + + + false + false + ${project.artifactId}-${project.version} + + hadoop-tools + + + + + + + + + diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml index 3166b9ef99..bc75e2b653 100644 --- a/hadoop-tools/pom.xml +++ b/hadoop-tools/pom.xml @@ -39,6 +39,7 @@ hadoop-datajoin hadoop-tools-dist hadoop-extras + hadoop-pipes From 0a18589197c3c556c2465493094d5b243738a805 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Tue, 19 Jun 2012 21:51:15 +0000 Subject: [PATCH 61/91] MAPREDUCE-4270. Move the data_join test classes to the correct path. (Contributed by Thomas Graves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351869 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../java/{ => org/apache/hadoop/contrib/utils/join}/README.txt | 0 .../hadoop/contrib/utils/join}/SampleDataJoinMapper.java | 0 .../hadoop/contrib/utils/join}/SampleDataJoinReducer.java | 0 .../hadoop/contrib/utils/join}/SampleTaggedMapOutput.java | 0 5 files changed, 3 insertions(+) rename hadoop-tools/hadoop-datajoin/src/test/java/{ => org/apache/hadoop/contrib/utils/join}/README.txt (100%) rename hadoop-tools/hadoop-datajoin/src/test/java/{ => org/apache/hadoop/contrib/utils/join}/SampleDataJoinMapper.java (100%) rename hadoop-tools/hadoop-datajoin/src/test/java/{ => org/apache/hadoop/contrib/utils/join}/SampleDataJoinReducer.java (100%) rename hadoop-tools/hadoop-datajoin/src/test/java/{ => org/apache/hadoop/contrib/utils/join}/SampleTaggedMapOutput.java (100%) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 29da54d5db..6e8e592ddb 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -589,6 +589,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4341. add types to capacity scheduler properties documentation (Karthik Kambatla via tgraves) + MAPREDUCE-4270. Move the data_join test classes to the correct path. + (Thomas Graves via sseth) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-tools/hadoop-datajoin/src/test/java/README.txt b/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/README.txt similarity index 100% rename from hadoop-tools/hadoop-datajoin/src/test/java/README.txt rename to hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/README.txt diff --git a/hadoop-tools/hadoop-datajoin/src/test/java/SampleDataJoinMapper.java b/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/SampleDataJoinMapper.java similarity index 100% rename from hadoop-tools/hadoop-datajoin/src/test/java/SampleDataJoinMapper.java rename to hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/SampleDataJoinMapper.java diff --git a/hadoop-tools/hadoop-datajoin/src/test/java/SampleDataJoinReducer.java b/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/SampleDataJoinReducer.java similarity index 100% rename from hadoop-tools/hadoop-datajoin/src/test/java/SampleDataJoinReducer.java rename to hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/SampleDataJoinReducer.java diff --git a/hadoop-tools/hadoop-datajoin/src/test/java/SampleTaggedMapOutput.java b/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/SampleTaggedMapOutput.java similarity index 100% rename from hadoop-tools/hadoop-datajoin/src/test/java/SampleTaggedMapOutput.java rename to hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/SampleTaggedMapOutput.java From 46242e779cef66e0d01518bc6fbb64470065418b Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Tue, 19 Jun 2012 22:00:45 +0000 Subject: [PATCH 62/91] MAPREDUCE-4306. Fix distributed shell to work with users other than the one running the daemons. (Contributed by Ahmed Radwan) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351876 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../distributedshell/ApplicationMaster.java | 11 ++++------- .../yarn/applications/distributedshell/Client.java | 14 ++------------ 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 6e8e592ddb..1f332e2b78 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -174,6 +174,9 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-3873. Fixed NodeManagers' decommissioning at RM to accept IP addresses also. (xieguiming via vinodkv) + MAPREDUCE-4306. Fix distributed shell to work with users other than the one + running the daemons. (Ahmed Radwan via sseth) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index b790e3c52a..592ab03822 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -44,7 +44,6 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.AMRMProtocol; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ContainerManager; @@ -635,12 +634,10 @@ public void run() { ctx.setContainerId(container.getId()); ctx.setResource(container.getResource()); - try { - ctx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); - } catch (IOException e) { - LOG.info("Getting current user info failed when trying to launch the container" - + e.getMessage()); - } + String jobUserName = System.getenv(ApplicationConstants.Environment.USER + .name()); + ctx.setUser(jobUserName); + LOG.info("Setting user in ContainerLaunchContext to: " + jobUserName); // Set the environment ctx.setEnvironment(shellEnv); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 41e8e6defc..55205f23ac 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -95,7 +95,7 @@ * *

For the actual job submission, the client first has to create an {@link ApplicationSubmissionContext}. * The {@link ApplicationSubmissionContext} defines the application details such as {@link ApplicationId} - * and application name, user submitting the application, the priority assigned to the application and the queue + * and application name, the priority assigned to the application and the queue * to which this application needs to be assigned. In addition to this, the {@link ApplicationSubmissionContext} * also defines the {@link ContainerLaunchContext} which describes the Container with which * the {@link ApplicationMaster} is launched.

@@ -132,8 +132,6 @@ public class Client { private int amPriority = 0; // Queue for App master private String amQueue = ""; - // User to run app master as - private String amUser = ""; // Amt. of memory resource to request for to run the App Master private int amMemory = 10; @@ -221,6 +219,7 @@ private void printUsage(Options opts) { * Parse command line options * @param args Parsed command line options * @return Whether the init was successful to run the client + * @throws ParseException */ public boolean init(String[] args) throws ParseException { @@ -228,7 +227,6 @@ public boolean init(String[] args) throws ParseException { opts.addOption("appname", true, "Application Name. Default value - DistributedShell"); opts.addOption("priority", true, "Application Priority. Default 0"); opts.addOption("queue", true, "RM Queue in which this application is to be submitted"); - opts.addOption("user", true, "User to run the application as"); opts.addOption("timeout", true, "Application timeout in milliseconds"); opts.addOption("master_memory", true, "Amount of memory in MB to be requested to run the application master"); opts.addOption("jar", true, "Jar file containing the application master"); @@ -263,7 +261,6 @@ public boolean init(String[] args) throws ParseException { appName = cliParser.getOptionValue("appname", "DistributedShell"); amPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0")); amQueue = cliParser.getOptionValue("queue", ""); - amUser = cliParser.getOptionValue("user", ""); amMemory = Integer.parseInt(cliParser.getOptionValue("master_memory", "10")); if (amMemory < 0) { @@ -567,10 +564,6 @@ else if (amMemory > maxMem) { commands.add(command.toString()); amContainer.setCommands(commands); - // For launching an AM Container, setting user here is not needed - // Set user in ApplicationSubmissionContext - // amContainer.setUser(amUser); - // Set up resource type requirements // For now, only memory is supported so we set memory requirements Resource capability = Records.newRecord(Resource.class); @@ -594,9 +587,6 @@ else if (amMemory > maxMem) { // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); - // Set the user submitting this application - // TODO can it be empty? - appContext.setUser(amUser); // Create the request to send to the applications manager SubmitApplicationRequest appRequest = Records.newRecord(SubmitApplicationRequest.class); From fc93b5adb9bdd73df398d4ed8c983b28f8b73102 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Wed, 20 Jun 2012 21:22:33 +0000 Subject: [PATCH 63/91] MAPREDUCE-3889. job client tries to use /tasklog interface, but that doesn't exist anymore (Devaraj K via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1352330 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../java/org/apache/hadoop/mapreduce/Job.java | 81 +------------------ 2 files changed, 4 insertions(+), 80 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 1f332e2b78..48c08b3efb 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -595,6 +595,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4270. Move the data_join test classes to the correct path. (Thomas Graves via sseth) + MAPREDUCE-3889. job client tries to use /tasklog interface, but that + doesn't exist anymore (Devaraj K via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java index 51bac98228..2fd666e827 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java @@ -18,30 +18,19 @@ package org.apache.hadoop.mapreduce; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.net.URL; -import java.net.URLConnection; import java.net.URI; import java.security.PrivilegedExceptionAction; -import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration.IntegerRanges; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.filecache.DistributedCache; @@ -1367,14 +1356,6 @@ private void printTaskEvents(TaskCompletionEvent[] events, Job.TaskStatusFilter filter, boolean profiling, IntegerRanges mapRanges, IntegerRanges reduceRanges) throws IOException, InterruptedException { for (TaskCompletionEvent event : events) { - TaskCompletionEvent.Status status = event.getStatus(); - if (profiling && shouldDownloadProfile() && - (status == TaskCompletionEvent.Status.SUCCEEDED || - status == TaskCompletionEvent.Status.FAILED) && - (event.isMapTask() ? mapRanges : reduceRanges). - isIncluded(event.idWithinJob())) { - downloadProfile(event); - } switch (filter) { case NONE: break; @@ -1382,7 +1363,6 @@ private void printTaskEvents(TaskCompletionEvent[] events, if (event.getStatus() == TaskCompletionEvent.Status.SUCCEEDED) { LOG.info(event.toString()); - displayTaskLogs(event.getTaskAttemptId(), event.getTaskTrackerHttp()); } break; case FAILED: @@ -1397,8 +1377,6 @@ private void printTaskEvents(TaskCompletionEvent[] events, System.err.println(diagnostics); } } - // Displaying the task logs - displayTaskLogs(event.getTaskAttemptId(), event.getTaskTrackerHttp()); } break; case KILLED: @@ -1408,67 +1386,10 @@ private void printTaskEvents(TaskCompletionEvent[] events, break; case ALL: LOG.info(event.toString()); - displayTaskLogs(event.getTaskAttemptId(), event.getTaskTrackerHttp()); break; } } } - - private void downloadProfile(TaskCompletionEvent e) throws IOException { - URLConnection connection = new URL( - getTaskLogURL(e.getTaskAttemptId(), e.getTaskTrackerHttp()) + - "&filter=profile").openConnection(); - InputStream in = connection.getInputStream(); - OutputStream out = new FileOutputStream(e.getTaskAttemptId() + ".profile"); - IOUtils.copyBytes(in, out, 64 * 1024, true); - } - - private void displayTaskLogs(TaskAttemptID taskId, String baseUrl) - throws IOException { - // The tasktracker for a 'failed/killed' job might not be around... - if (baseUrl != null) { - // Construct the url for the tasklogs - String taskLogUrl = getTaskLogURL(taskId, baseUrl); - - // Copy tasks's stdout of the JobClient - getTaskLogs(taskId, new URL(taskLogUrl+"&filter=stdout"), System.out); - - // Copy task's stderr to stderr of the JobClient - getTaskLogs(taskId, new URL(taskLogUrl+"&filter=stderr"), System.err); - } - } - - private void getTaskLogs(TaskAttemptID taskId, URL taskLogUrl, - OutputStream out) { - try { - int tasklogtimeout = cluster.getConf().getInt( - TASKLOG_PULL_TIMEOUT_KEY, DEFAULT_TASKLOG_TIMEOUT); - URLConnection connection = taskLogUrl.openConnection(); - connection.setReadTimeout(tasklogtimeout); - connection.setConnectTimeout(tasklogtimeout); - BufferedReader input = - new BufferedReader(new InputStreamReader(connection.getInputStream())); - BufferedWriter output = - new BufferedWriter(new OutputStreamWriter(out)); - try { - String logData = null; - while ((logData = input.readLine()) != null) { - if (logData.length() > 0) { - output.write(taskId + ": " + logData + "\n"); - output.flush(); - } - } - } finally { - input.close(); - } - } catch(IOException ioe) { - LOG.warn("Error reading task output " + ioe.getMessage()); - } - } - - private String getTaskLogURL(TaskAttemptID taskId, String baseUrl) { - return (baseUrl + "/tasklog?plaintext=true&attemptid=" + taskId); - } /** The interval at which monitorAndPrintJob() prints status */ public static int getProgressPollInterval(Configuration conf) { From 9a3e3819ab90afa8c71da84b0429b81f3ef03d64 Mon Sep 17 00:00:00 2001 From: Ravi Gummadi Date: Thu, 21 Jun 2012 05:44:20 +0000 Subject: [PATCH 64/91] MAPREDUCE-4356. [Rumen] Provide access to the method ParsedTask.obtainTaskAttempts(). (ravigummadi) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1352415 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/tools/rumen/ParsedTask.java | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 48c08b3efb..df79e09486 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -60,6 +60,9 @@ Trunk (unreleased changes) BUG FIXES + MAPREDUCE-4356. [Rumen] Provide access to the method + ParsedTask.obtainTaskAttempts(). (ravigummadi) + MAPREDUCE-4100. [Gridmix] Bug fixed in compression emulation feature for map only jobs. (amarrk) diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ParsedTask.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ParsedTask.java index 90eebd66fb..11cf12d042 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ParsedTask.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ParsedTask.java @@ -91,7 +91,10 @@ public String obtainFailedDueToAttemptId() { return failedDueToAttempt; } - List obtainTaskAttempts() { + /** + * @return the list of attempts of this task. + */ + public List obtainTaskAttempts() { List attempts = getAttempts(); return convertTaskAttempts(attempts); } From 915d1b52fc83787be78c7688ab3bf24dc67d4e46 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Thu, 21 Jun 2012 15:01:35 +0000 Subject: [PATCH 65/91] MAPREDUCE-4320. gridmix mainClass wrong in pom.xml (tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1352550 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ hadoop-tools/hadoop-gridmix/pom.xml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index df79e09486..a8f8633d72 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -601,6 +601,8 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-3889. job client tries to use /tasklog interface, but that doesn't exist anymore (Devaraj K via bobby) + MAPREDUCE-4320. gridmix mainClass wrong in pom.xml (tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml index 9537883464..15ed600324 100644 --- a/hadoop-tools/hadoop-gridmix/pom.xml +++ b/hadoop-tools/hadoop-gridmix/pom.xml @@ -121,7 +121,7 @@ - org.apache.hadoop.tools.HadoopArchives + org.apache.hadoop.mapred.gridmix.Gridmix From ab17f633e6e3d4e146b80851d7f562470d5edd2a Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Thu, 21 Jun 2012 18:14:22 +0000 Subject: [PATCH 66/91] MAPREDUCE-4295. RM crashes due to DNS issue (tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1352638 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../scheduler/capacity/LeafQueue.java | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a8f8633d72..54e74a9594 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -603,6 +603,8 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4320. gridmix mainClass wrong in pom.xml (tgraves) + MAPREDUCE-4295. RM crashes due to DNS issue (tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 75d52490d6..aed6c902fb 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -1180,9 +1180,16 @@ public Container createContainer(SchedulerApp application, SchedulerNode node, if (UserGroupInformation.isSecurityEnabled()) { ContainerTokenIdentifier tokenIdentifier = new ContainerTokenIdentifier( containerId, nodeId.toString(), capability); - containerToken = BuilderUtils.newContainerToken(nodeId, ByteBuffer - .wrap(containerTokenSecretManager - .createPassword(tokenIdentifier)), tokenIdentifier); + try { + containerToken = BuilderUtils.newContainerToken(nodeId, ByteBuffer + .wrap(containerTokenSecretManager + .createPassword(tokenIdentifier)), tokenIdentifier); + } catch (IllegalArgumentException e) { + // this could be because DNS is down - in which case we just want + // to retry and not bring RM down + LOG.error("Error trying to create new container", e); + return null; + } } // Create the container @@ -1211,6 +1218,11 @@ private Resource assignContainer(Resource clusterResource, SchedulerNode node, // Create the container if necessary Container container = getContainer(rmContainer, application, node, capability, priority); + + // something went wrong getting/creating the container + if (container == null) { + return Resources.none(); + } // Can we allocate a container on this node? int availableContainers = From f9d5618f8d6d97a2868f9a83f03b992272fefb2e Mon Sep 17 00:00:00 2001 From: Harsh J Date: Thu, 21 Jun 2012 21:12:31 +0000 Subject: [PATCH 67/91] HADOOP-8524. Allow users to get source of a Configuration parameter. (harsh) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1352689 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../org/apache/hadoop/conf/Configuration.java | 32 +++++++++++++++++++ .../apache/hadoop/conf/TestConfiguration.java | 20 ++++++++++++ 3 files changed, 55 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ecaa1bf9d2..1fff076215 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -209,6 +209,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8368. Use CMake rather than autotools to build native code (ccccabe via tucu) + HADOOP-8524. Allow users to get source of a Configuration + parameter (harsh) + BUG FIXES HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 917f97c69b..c54070eaaf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -1070,6 +1070,38 @@ public void setPattern(String name, Pattern pattern) { } } + /** + * Gets the absolute path to the resource object (file, URL, etc.), for a given + * property name. + * + * @param name - The property name to get the source of. + * @return null - If the property or its source wasn't found or if the property + * was defined in code (i.e. in a Configuration instance, not from a physical + * resource). Otherwise, returns the absolute path of the resource that loaded + * the property name, as a String. + */ + @InterfaceStability.Unstable + public synchronized String getPropertySource(String name) { + if (properties == null) { + // If properties is null, it means a resource was newly added + // but the props were cleared so as to load it upon future + // requests. So lets force a load by asking a properties list. + getProps(); + } + // Return a null right away if our properties still + // haven't loaded or the resource mapping isn't defined + if (properties == null || updatingResource == null) { + return null; + } else { + String source = updatingResource.get(name); + if (source == null || source.equals(UNKNOWN_RESOURCE)) { + return null; + } else { + return source; + } + } + } + /** * A class that represents a set of positive integer ranges. It parses * strings of the form: "2-3,5,7-" where ranges are separated by comma and diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 34a1780142..4878031262 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -663,6 +663,26 @@ public void testPattern() throws IOException { conf.getPattern("test.pattern3", defaultPattern).pattern()); } + public void testPropertySource() throws IOException { + out = new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendProperty("test.foo", "bar"); + endConfig(); + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + conf.set("fs.defaultFS", "value"); + assertEquals( + "Resource string returned for a file-loaded property" + + " must be a proper absolute path", + fileResource, + new Path(conf.getPropertySource("test.foo"))); + assertEquals("Resource string returned for a set() property must be null", + null, + conf.getPropertySource("fs.defaultFS")); + assertEquals("Resource string returned for an unset property must be null", + null, conf.getPropertySource("fs.defaultFoo")); + } + public void testSocketAddress() throws IOException { Configuration conf = new Configuration(); final String defaultAddr = "host:1"; From ba579b792a6dc60fe2da8aeebdd23b36bd801743 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Fri, 22 Jun 2012 21:40:35 +0000 Subject: [PATCH 68/91] MAPREDUCE-4031. Prevent a Node Manager hang during shutdown. (Contributed by Devaraj K) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1353038 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/util/ShutdownHookManager.java | 13 +++++++++++-- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../apache/hadoop/yarn/event/AsyncDispatcher.java | 4 +++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java index f907e3efeb..989c96a8e3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java @@ -30,7 +30,7 @@ /** * The ShutdownHookManager enables running shutdownHook - * in a determistic order, higher priority first. + * in a deterministic order, higher priority first. *

* The JVM runs ShutdownHooks in a non-deterministic order or in parallel. * This class registers a single JVM shutdownHook and run all the @@ -169,7 +169,7 @@ public boolean removeShutdownHook(Runnable shutdownHook) { } /** - * Indicates if a shutdownHook is registered or nt. + * Indicates if a shutdownHook is registered or not. * * @param shutdownHook shutdownHook to check if registered. * @return TRUE/FALSE depending if the shutdownHook is is registered. @@ -177,5 +177,14 @@ public boolean removeShutdownHook(Runnable shutdownHook) { public boolean hasShutdownHook(Runnable shutdownHook) { return hooks.contains(new HookEntry(shutdownHook, 0)); } + + /** + * Indicates if shutdown is in progress or not. + * + * @return TRUE if the shutdown is in progress, otherwise FALSE. + */ + public boolean isShutdownInProgress() { + return shutdownInProgress.get(); + } } diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 54e74a9594..c5943335a6 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -605,6 +605,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4295. RM crashes due to DNS issue (tgraves) + MAPREDUCE-4031. Prevent a Node Manager hang during shutdown. + (Devaraj K via sseth) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java index 1b3a76a477..c8f325df24 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.service.AbstractService; @@ -127,7 +128,8 @@ protected void dispatch(Event event) { catch (Throwable t) { //TODO Maybe log the state of the queue LOG.fatal("Error in dispatcher thread", t); - if (exitOnDispatchException) { + if (exitOnDispatchException + && (ShutdownHookManager.get().isShutdownInProgress()) == false) { LOG.info("Exiting, bbye.."); System.exit(-1); } From f992e5112d9b9492c31e2cb1b523d3ee3c614004 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Fri, 22 Jun 2012 21:45:59 +0000 Subject: [PATCH 69/91] MAPREDUCE-4031 amendment. Fixes CHANGES.txt to reflect correct fix version. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1353039 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index c5943335a6..352378a530 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -180,6 +180,9 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-4306. Fix distributed shell to work with users other than the one running the daemons. (Ahmed Radwan via sseth) + MAPREDUCE-4031. Prevent a Node Manager hang during shutdown. + (Devaraj K via sseth) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES @@ -605,9 +608,6 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4295. RM crashes due to DNS issue (tgraves) - MAPREDUCE-4031. Prevent a Node Manager hang during shutdown. - (Devaraj K via sseth) - Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES From c8030899bc853e1bb8a92e676c7308d5df2fecb4 Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Mon, 25 Jun 2012 15:28:09 +0000 Subject: [PATCH 70/91] HDFS-3550. Fix raid javadoc warnings. (Jason Lowe via daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1353592 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/main/java/org/apache/hadoop/raid/Decoder.java | 4 ++-- .../main/java/org/apache/hadoop/raid/DistRaidNode.java | 8 ++++---- .../src/main/java/org/apache/hadoop/raid/Encoder.java | 1 - .../apache/hadoop/raid/RaidConfigurationException.java | 2 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Decoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Decoder.java index c38885ce9d..65060085c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Decoder.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Decoder.java @@ -166,9 +166,9 @@ public void decodeFile( * * @param srcFs The filesystem containing the source file. * @param srcPath The damaged source file. - * @param parityPath The filesystem containing the parity file. This could be + * @param parityFs The filesystem containing the parity file. This could be * different from fs in case the parity file is part of a HAR archive. - * @param parityFile The parity file. + * @param parityPath The parity file. * @param blockSize The block size of the file. * @param blockOffset Known location of error in the source file. There could * be additional errors in the source file that are discovered during diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaidNode.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaidNode.java index d2219ae745..43be2a725d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaidNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaidNode.java @@ -52,7 +52,7 @@ public DistRaidNode(Configuration conf) throws IOException { } /** - * {@inheritDocs} + * {@inheritDoc} */ @Override public void join() { @@ -65,7 +65,7 @@ public void join() { } /** - * {@inheritDocs} + * {@inheritDoc} */ @Override public void stop() { @@ -79,7 +79,7 @@ public void stop() { /** - * {@inheritDocs} + * {@inheritDoc} */ @Override void raidFiles(PolicyInfo info, List paths) throws IOException { @@ -95,7 +95,7 @@ void raidFiles(PolicyInfo info, List paths) throws IOException { } /** - * {@inheritDocs} + * {@inheritDoc} */ @Override int getRunningJobsForPolicy(String policyName) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Encoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Encoder.java index f74b3a025d..b87bad4f80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Encoder.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Encoder.java @@ -331,7 +331,6 @@ protected InputStream[] stripeInputs( * The implementation of generating parity data for a stripe. * * @param blocks The streams to blocks in the stripe. - * @param srcFile The source file. * @param stripeStartOffset The start offset of the stripe * @param blockSize The maximum size of a block. * @param outs output streams to the parity blocks. diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidConfigurationException.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidConfigurationException.java index 7335999ada..891863ae92 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidConfigurationException.java +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidConfigurationException.java @@ -19,7 +19,7 @@ package org.apache.hadoop.raid; /** - * Thrown when the config file for {@link CronNode} is malformed. + * Thrown when the config file for {@link RaidNode} is malformed. */ public class RaidConfigurationException extends Exception { private static final long serialVersionUID = 4046516718965587999L; diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index fc632047c8..954b8ec1da 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -154,6 +154,8 @@ Trunk (unreleased changes) HDFS-3462. TestDFSClientRetries.busyTest() should restore default xceiver count in the config. (Madhukara Phatak via harsh) + HDFS-3550. Fix raid javadoc warnings. (Jason Lowe via daryn) + Branch-2 ( Unreleased changes ) INCOMPATIBLE CHANGES From c7f872543efa6711b7ec8ef5285afe3094b0dbba Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 25 Jun 2012 15:58:55 +0000 Subject: [PATCH 71/91] MAPREDUCE-4336. Distributed Shell fails when used with the CapacityScheduler (ahmed via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1353625 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/yarn/applications/distributedshell/Client.java | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 352378a530..9a80dddedf 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -183,6 +183,9 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-4031. Prevent a Node Manager hang during shutdown. (Devaraj K via sseth) + MAPREDUCE-4336. Distributed Shell fails when used with the CapacityScheduler + (ahmed via tucu) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 55205f23ac..bfea87af42 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -260,7 +260,7 @@ public boolean init(String[] args) throws ParseException { appName = cliParser.getOptionValue("appname", "DistributedShell"); amPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0")); - amQueue = cliParser.getOptionValue("queue", ""); + amQueue = cliParser.getOptionValue("queue", "default"); amMemory = Integer.parseInt(cliParser.getOptionValue("master_memory", "10")); if (amMemory < 0) { @@ -353,6 +353,7 @@ public boolean run() throws IOException { } GetQueueInfoRequest queueInfoReq = Records.newRecord(GetQueueInfoRequest.class); + queueInfoReq.setQueueName(this.amQueue); GetQueueInfoResponse queueInfoResp = applicationsManager.getQueueInfo(queueInfoReq); QueueInfo queueInfo = queueInfoResp.getQueueInfo(); LOG.info("Queue info" From 5de582ae191f9ec7655fba46a5b979d6722f2fe8 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 25 Jun 2012 16:02:39 +0000 Subject: [PATCH 72/91] Adding svn-ignore for hdfs-raid and pipe target dirs git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1353627 13f79535-47bb-0310-9956-ffa450edef68 From 09078f15defd7bc8d87dc54af5e54eb7e58ca11c Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Mon, 25 Jun 2012 17:54:48 +0000 Subject: [PATCH 73/91] MAPREDUCE-4290. Fix Yarn Applicaiton Status to MR JobState conversion. (Contributed by Devaraj K) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1353684 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../hadoop/mapreduce/TypeConverter.java | 18 +++-- .../hadoop/mapreduce/TestTypeConverter.java | 5 +- .../mapred/TestResourceMgrDelegate.java | 67 ++++++++++++++++++- 4 files changed, 83 insertions(+), 10 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 9a80dddedf..bc4a33d73c 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -186,6 +186,9 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-4336. Distributed Shell fails when used with the CapacityScheduler (ahmed via tucu) + MAPREDUCE-4290. Fix Yarn Applicaiton Status to MR JobState conversion. + (Devaraj K via sseth) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java index ddabb4c52f..12dce4b901 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueState; @@ -376,22 +377,27 @@ public static List fromYarn( } return reports; } - - public static JobStatus.State fromYarn(YarnApplicationState state) { - switch (state) { + + public static State fromYarn(YarnApplicationState yarnApplicationState, + FinalApplicationStatus finalApplicationStatus) { + switch (yarnApplicationState) { case NEW: case SUBMITTED: return State.PREP; case RUNNING: return State.RUNNING; case FINISHED: - return State.SUCCEEDED; + if (finalApplicationStatus == FinalApplicationStatus.SUCCEEDED) { + return State.SUCCEEDED; + } else if (finalApplicationStatus == FinalApplicationStatus.KILLED) { + return State.KILLED; + } case FAILED: return State.FAILED; case KILLED: return State.KILLED; } - throw new YarnException("Unrecognized application state: " + state); + throw new YarnException("Unrecognized application state: " + yarnApplicationState); } private static final String TT_NAME_PREFIX = "tracker_"; @@ -417,7 +423,7 @@ public static JobStatus fromYarn(ApplicationReport application, new JobStatus( TypeConverter.fromYarn(application.getApplicationId()), 0.0f, 0.0f, 0.0f, 0.0f, - TypeConverter.fromYarn(application.getYarnApplicationState()), + TypeConverter.fromYarn(application.getYarnApplicationState(), application.getFinalApplicationStatus()), org.apache.hadoop.mapreduce.JobPriority.NORMAL, application.getUser(), application.getName(), application.getQueue(), jobFile, trackingUrl, false diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java index a9a2c0a9b9..49dec4a3b7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java @@ -27,6 +27,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; @@ -45,7 +46,7 @@ public class TestTypeConverter { @Test public void testEnums() throws Exception { for (YarnApplicationState applicationState : YarnApplicationState.values()) { - TypeConverter.fromYarn(applicationState); + TypeConverter.fromYarn(applicationState, FinalApplicationStatus.FAILED); } for (TaskType taskType : TaskType.values()) { @@ -63,8 +64,6 @@ public void testEnums() throws Exception { for (TaskState taskState : TaskState.values()) { TypeConverter.fromYarn(taskState); } - - } @Test diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java index 5334f29c9b..ad1ebc9d80 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java @@ -19,13 +19,26 @@ package org.apache.hadoop.mapred; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import junit.framework.Assert; +import org.apache.hadoop.mapreduce.JobStatus; +import org.apache.hadoop.mapreduce.JobStatus.State; import org.apache.hadoop.yarn.api.ClientRMProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.Records; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; @@ -35,7 +48,7 @@ public class TestResourceMgrDelegate { /** * Tests that getRootQueues makes a request for the (recursive) child queues */ -@Test + @Test public void testGetRootQueues() throws IOException, InterruptedException { ClientRMProtocol applicationsManager = Mockito.mock(ClientRMProtocol.class); GetQueueInfoResponse response = Mockito.mock(GetQueueInfoResponse.class); @@ -60,4 +73,56 @@ public void testGetRootQueues() throws IOException, InterruptedException { argument.getValue().getRecursive()); } + @Test + public void tesAllJobs() throws Exception { + ClientRMProtocol applicationsManager = Mockito.mock(ClientRMProtocol.class); + GetAllApplicationsResponse allApplicationsResponse = Records + .newRecord(GetAllApplicationsResponse.class); + List applications = new ArrayList(); + applications.add(getApplicationReport(YarnApplicationState.FINISHED, + FinalApplicationStatus.FAILED)); + applications.add(getApplicationReport(YarnApplicationState.FINISHED, + FinalApplicationStatus.SUCCEEDED)); + applications.add(getApplicationReport(YarnApplicationState.FINISHED, + FinalApplicationStatus.KILLED)); + applications.add(getApplicationReport(YarnApplicationState.FAILED, + FinalApplicationStatus.FAILED)); + allApplicationsResponse.setApplicationList(applications); + Mockito.when( + applicationsManager.getAllApplications(Mockito + .any(GetAllApplicationsRequest.class))).thenReturn( + allApplicationsResponse); + ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate( + new YarnConfiguration(), applicationsManager); + JobStatus[] allJobs = resourceMgrDelegate.getAllJobs(); + + Assert.assertEquals(State.FAILED, allJobs[0].getState()); + Assert.assertEquals(State.SUCCEEDED, allJobs[1].getState()); + Assert.assertEquals(State.KILLED, allJobs[2].getState()); + Assert.assertEquals(State.FAILED, allJobs[3].getState()); + } + + private ApplicationReport getApplicationReport( + YarnApplicationState yarnApplicationState, + FinalApplicationStatus finalApplicationStatus) { + ApplicationReport appReport = Mockito.mock(ApplicationReport.class); + ApplicationResourceUsageReport appResources = Mockito + .mock(ApplicationResourceUsageReport.class); + Mockito.when(appReport.getApplicationId()).thenReturn( + Records.newRecord(ApplicationId.class)); + Mockito.when(appResources.getNeededResources()).thenReturn( + Records.newRecord(Resource.class)); + Mockito.when(appResources.getReservedResources()).thenReturn( + Records.newRecord(Resource.class)); + Mockito.when(appResources.getUsedResources()).thenReturn( + Records.newRecord(Resource.class)); + Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn( + appResources); + Mockito.when(appReport.getYarnApplicationState()).thenReturn( + yarnApplicationState); + Mockito.when(appReport.getFinalApplicationStatus()).thenReturn( + finalApplicationStatus); + + return appReport; + } } From 54da505832139481a4e18f760648fee3affe9d05 Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Mon, 25 Jun 2012 18:26:29 +0000 Subject: [PATCH 74/91] HDFS-3549. Fix dist tar build fails in hadoop-hdfs-raid project. (Jason Lowe via daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1353695 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs-raid/pom.xml | 36 ++------------------ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ 2 files changed, 4 insertions(+), 34 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-raid/pom.xml index f4012b93f2..a7fa6eb656 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-raid/pom.xml @@ -97,7 +97,7 @@ org.codehaus.mojo findbugs-maven-plugin - ${basedir}/dev-support/findbugsExcludeFile.xml + @@ -148,7 +148,7 @@ dist - package + prepare-package single @@ -163,38 +163,6 @@ - - org.apache.maven.plugins - maven-antrun-plugin - - - tar - package - - run - - - - - - - which cygpath 2> /dev/null - if [ $? = 1 ]; then - BUILD_DIR="${project.build.directory}" - else - BUILD_DIR=`cygpath --unix '${project.build.directory}'` - fi - cd $BUILD_DIR - tar czf ${project.artifactId}-${project.version}.tar.gz ${project.artifactId}-${project.version} - - - - - - - - - diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 954b8ec1da..9e4d42b751 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -156,6 +156,8 @@ Trunk (unreleased changes) HDFS-3550. Fix raid javadoc warnings. (Jason Lowe via daryn) + HDFS-3549. Fix dist tar build fails in hadoop-hdfs-raid project. (Jason Lowe via daryn) + Branch-2 ( Unreleased changes ) INCOMPATIBLE CHANGES From 64b3044d6e4068f93b0f1f08f32c74aa00555e89 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 25 Jun 2012 21:36:17 +0000 Subject: [PATCH 75/91] MAPREDUCE-2289. Permissions race can make getStagingDir fail on local filesystem (ahmed via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1353750 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapreduce/JobSubmissionFiles.java | 26 +++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index bc4a33d73c..ab014f6356 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -189,6 +189,9 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-4290. Fix Yarn Applicaiton Status to MR JobState conversion. (Devaraj K via sseth) + MAPREDUCE-2289. Permissions race can make getStagingDir fail on local filesystem + (ahmed via tucu) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmissionFiles.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmissionFiles.java index b084d1ccde..a4ea1d80a0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmissionFiles.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmissionFiles.java @@ -27,12 +27,18 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + /** * A utility to manage job submission files. */ @InterfaceAudience.Private public class JobSubmissionFiles { + private final static Log LOG = LogFactory.getLog(JobSubmissionFiles.class); + // job submission directory is private! final public static FsPermission JOB_DIR_PERMISSION = FsPermission.createImmutable((short) 0700); // rwx-------- @@ -102,14 +108,18 @@ public static Path getStagingDir(Cluster cluster, Configuration conf) if (fs.exists(stagingArea)) { FileStatus fsStatus = fs.getFileStatus(stagingArea); String owner = fsStatus.getOwner(); - if (!(owner.equals(currentUser) || owner.equals(realUser)) || - !fsStatus.getPermission().equals(JOB_DIR_PERMISSION)) { - throw new IOException("The ownership/permissions on the staging " + - "directory " + stagingArea + " is not as expected. " + - "It is owned by " + owner + " and permissions are "+ - fsStatus.getPermission() + ". The directory must " + + if (!(owner.equals(currentUser) || owner.equals(realUser))) { + throw new IOException("The ownership on the staging directory " + + stagingArea + " is not as expected. " + + "It is owned by " + owner + ". The directory must " + "be owned by the submitter " + currentUser + " or " + - "by " + realUser + " and permissions must be rwx------"); + "by " + realUser); + } + if (!fsStatus.getPermission().equals(JOB_DIR_PERMISSION)) { + LOG.info("Permissions on staging directory " + stagingArea + " are " + + "incorrect: " + fsStatus.getPermission() + ". Fixing permissions " + + "to correct value " + JOB_DIR_PERMISSION); + fs.setPermission(stagingArea, JOB_DIR_PERMISSION); } } else { fs.mkdirs(stagingArea, @@ -118,4 +128,4 @@ public static Path getStagingDir(Cluster cluster, Configuration conf) return stagingArea; } -} \ No newline at end of file +} From 3610da469ac4d9d80b81f75624ed98559417dfdb Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 25 Jun 2012 21:55:44 +0000 Subject: [PATCH 76/91] MAPREDUCE-4355. Add JobStatus getJobStatus(JobID) to JobClient. (kkambatl via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1353757 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 + ...obClientGetJob.java => TestJobClient.java} | 22 ++++++++- .../org/apache/hadoop/mapred/JobClient.java | 47 ++++++++++++++----- .../org/apache/hadoop/mapreduce/Cluster.java | 15 +++++- 4 files changed, 70 insertions(+), 16 deletions(-) rename hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/{TestJobClientGetJob.java => TestJobClient.java} (71%) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ab014f6356..5cce59ba04 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -129,6 +129,8 @@ Branch-2 ( Unreleased changes ) NEW FEATURES + MAPREDUCE-4355. Add JobStatus getJobStatus(JobID) to JobClient. (kkambatl via tucu) + IMPROVEMENTS MAPREDUCE-4146. Support limits on task status string length and number of diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClient.java similarity index 71% rename from hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java rename to hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClient.java index f929e4f4a8..87afeb326c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClient.java @@ -19,6 +19,7 @@ package org.apache.hadoop.mapred; import static junit.framework.Assert.assertNotNull; +import static org.junit.Assert.assertEquals; import java.io.IOException; @@ -28,7 +29,7 @@ import org.apache.hadoop.fs.Path; import org.junit.Test; -public class TestJobClientGetJob { +public class TestJobClient { private static Path TEST_ROOT_DIR = new Path(System.getProperty("test.build.data","/tmp")); @@ -45,7 +46,7 @@ private Path createTempFile(String filename, String contents) @SuppressWarnings("deprecation") @Test - public void testGetRunningJobFromJobClient() throws Exception { + public void testGetRunningJob() throws Exception { JobConf conf = new JobConf(); conf.set("mapreduce.framework.name", "local"); FileInputFormat.addInputPath(conf, createTempFile("in", "hello")); @@ -60,4 +61,21 @@ public void testGetRunningJobFromJobClient() throws Exception { assertNotNull("New running job", newRunningJob); } + @SuppressWarnings("deprecation") + @Test + public void testGetJobStatus() throws Exception { + JobConf conf = new JobConf(); + conf.set("mapreduce.framework.name", "local"); + FileInputFormat.addInputPath(conf, createTempFile("in", "hello")); + Path outputDir = new Path(TEST_ROOT_DIR, getClass().getSimpleName()); + outputDir.getFileSystem(conf).delete(outputDir, true); + FileOutputFormat.setOutputPath(conf, outputDir); + JobClient jc = new JobClient(conf); + RunningJob runningJob = jc.submitJob(conf); + assertNotNull("Running job", runningJob); + JobID jobid = runningJob.getID(); + JobStatus jobStatus = jc.getJobStatus(jobid); + assertNotNull("New running job", jobStatus); + assertEquals("Equal JobIDs", jobid, jobStatus.getJobID()); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java index 5d7919dbea..0bc4eaea8f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java @@ -620,6 +620,15 @@ public Job run() throws IOException, ClassNotFoundException, } } + private JobStatus getJobStatusUsingCluster(final JobID jobId) + throws IOException, InterruptedException { + return clientUgi.doAs(new PrivilegedExceptionAction() { + public JobStatus run() throws IOException, InterruptedException { + return JobStatus.downgrade(cluster.getJobStatus(jobId)); + } + }); + } + private Job getJobUsingCluster(final JobID jobid) throws IOException, InterruptedException { return clientUgi.doAs(new PrivilegedExceptionAction() { @@ -628,28 +637,40 @@ public Job run() throws IOException, InterruptedException { } }); } + /** - * Get an {@link RunningJob} object to track an ongoing job. Returns - * null if the id does not correspond to any known job. + * Get {@link JobStatus} of a job. Returns null if the id does not correspond + * to any known job. * - * @param jobid the jobid of the job. - * @return the {@link RunningJob} handle to track the job, null if the + * @param jobid + * the jobid of the job. + * @return the {@link JobStatus} object to retrieve the job stats, null if the * jobid doesn't correspond to any known job. * @throws IOException */ - public RunningJob getJob(final JobID jobid) throws IOException { + public JobStatus getJobStatus(JobID jobId) throws IOException { try { - - Job job = getJobUsingCluster(jobid); - if (job != null) { - JobStatus status = JobStatus.downgrade(job.getStatus()); - if (status != null) { - return new NetworkedJob(status, cluster); - } - } + return getJobStatusUsingCluster(jobId); } catch (InterruptedException ie) { throw new IOException(ie); } + } + + /** + * Get an {@link RunningJob} object to track an ongoing job. Returns null if + * the id does not correspond to any known job. + * + * @param jobid + * the jobid of the job. + * @return the {@link RunningJob} handle to track the job, null if the + * jobid doesn't correspond to any known job. + * @throws IOException + */ + public RunningJob getJob(JobID jobId) throws IOException { + JobStatus status = getJobStatus(jobId); + if (status != null) { + return new NetworkedJob(status, cluster); + } return null; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java index e456a7afa8..a3aee69550 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java @@ -172,6 +172,19 @@ public FileSystem run() throws IOException, InterruptedException { return fs; } + /** + * Get JobStatus corresponding to jobId. + * + * @param jobId + * @return object of {@link JobStatus} + * @throws IOException + * @throws InterruptedException + */ + public JobStatus getJobStatus(JobID jobId) throws IOException, + InterruptedException { + return client.getJobStatus(jobId); + } + /** * Get job corresponding to jobid. * @@ -181,7 +194,7 @@ public FileSystem run() throws IOException, InterruptedException { * @throws InterruptedException */ public Job getJob(JobID jobId) throws IOException, InterruptedException { - JobStatus status = client.getJobStatus(jobId); + JobStatus status = getJobStatus(jobId); if (status != null) { return Job.getInstance(this, status, new JobConf(status.getJobFile())); } From f9f5953c234af26a92f63b1d39c1f0c6409013fa Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 26 Jun 2012 01:58:13 +0000 Subject: [PATCH 77/91] HDFS-3516. Check content-type in WebHdfsFileSystem. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1353800 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hadoop/hdfs/web/WebHdfsFileSystem.java | 24 +++++++++++++++---- .../web/TestWebHdfsFileSystemContract.java | 20 ++++++++++++++++ .../hadoop/hdfs/web/WebHdfsTestUtil.java | 4 ++-- 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9e4d42b751..7acb002ee3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -238,6 +238,8 @@ Branch-2 ( Unreleased changes ) HDFS-3372. offlineEditsViewer should be able to read a binary edits file with recovery mode. (Colin Patrick McCabe via eli) + HDFS-3516. Check content-type in WebHdfsFileSystem. (szetszwo) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 7dbaded8d6..4327dd18c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -34,6 +34,8 @@ import java.util.Map; import java.util.StringTokenizer; +import javax.ws.rs.core.MediaType; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -252,9 +254,23 @@ private Path makeAbsolute(Path f) { return f.isAbsolute()? f: new Path(workingDir, f); } - static Map jsonParse(final InputStream in) throws IOException { + static Map jsonParse(final HttpURLConnection c, final boolean useErrorStream + ) throws IOException { + if (c.getContentLength() == 0) { + return null; + } + final InputStream in = useErrorStream? c.getErrorStream(): c.getInputStream(); if (in == null) { - throw new IOException("The input stream is null."); + throw new IOException("The " + (useErrorStream? "error": "input") + " stream is null."); + } + final String contentType = c.getContentType(); + if (contentType != null) { + final MediaType parsed = MediaType.valueOf(contentType); + if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) { + throw new IOException("Content-Type \"" + contentType + + "\" is incompatible with \"" + MediaType.APPLICATION_JSON + + "\" (parsed=\"" + parsed + "\")"); + } } return (Map)JSON.parse(new InputStreamReader(in)); } @@ -265,7 +281,7 @@ private Path makeAbsolute(Path f) { if (code != op.getExpectedHttpResponseCode()) { final Map m; try { - m = jsonParse(conn.getErrorStream()); + m = jsonParse(conn, true); } catch(IOException e) { throw new IOException("Unexpected HTTP response: code=" + code + " != " + op.getExpectedHttpResponseCode() + ", " + op.toQueryString() @@ -425,7 +441,7 @@ static HttpURLConnection twoStepWrite(HttpURLConnection conn, final HttpURLConnection conn = httpConnect(op, fspath, parameters); try { final Map m = validateResponse(op, conn); - return m != null? m: jsonParse(conn.getInputStream()); + return m != null? m: jsonParse(conn, false); } finally { conn.disconnect(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java index 61734180cf..b49818cd68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java @@ -287,6 +287,10 @@ public void testResponseCode() throws IOException { final Path root = new Path("/"); final Path dir = new Path("/test/testUrl"); assertTrue(webhdfs.mkdirs(dir)); + final Path file = new Path("/test/file"); + final FSDataOutputStream out = webhdfs.create(file); + out.write(1); + out.close(); {//test GETHOMEDIRECTORY final URL url = webhdfs.toUrl(GetOpParam.Op.GETHOMEDIRECTORY, root); @@ -378,5 +382,21 @@ public void testResponseCode() throws IOException { conn.connect(); assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); } + + {//test jsonParse with non-json type. + final HttpOpParam.Op op = GetOpParam.Op.OPEN; + final URL url = webhdfs.toUrl(op, file); + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod(op.getType().toString()); + conn.connect(); + + try { + WebHdfsFileSystem.jsonParse(conn, false); + fail(); + } catch(IOException ioe) { + WebHdfsFileSystem.LOG.info("GOOD", ioe); + } + conn.disconnect(); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java index 9ea0a4684a..38e2168519 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java @@ -49,7 +49,7 @@ public static WebHdfsFileSystem getWebHdfsFileSystem(final Configuration conf public static WebHdfsFileSystem getWebHdfsFileSystemAs( final UserGroupInformation ugi, final Configuration conf - ) throws IOException, URISyntaxException, InterruptedException { + ) throws IOException, InterruptedException { return ugi.doAs(new PrivilegedExceptionAction() { @Override public WebHdfsFileSystem run() throws Exception { @@ -70,7 +70,7 @@ public static URL toUrl(final WebHdfsFileSystem webhdfs, final int expectedResponseCode) throws IOException { conn.connect(); Assert.assertEquals(expectedResponseCode, conn.getResponseCode()); - return WebHdfsFileSystem.jsonParse(conn.getInputStream()); + return WebHdfsFileSystem.jsonParse(conn, false); } public static HttpURLConnection twoStepWrite(HttpURLConnection conn, From cd00688d3b6f9d8fd9e1b0b736b0b674ed2a17be Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 26 Jun 2012 03:25:47 +0000 Subject: [PATCH 78/91] HDFS-3498. Support replica removal in BlockPlacementPolicy and make BlockPlacementPolicyDefault extensible for reusing code in subclasses. Contributed by Junping Du git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1353807 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 + .../server/blockmanagement/BlockManager.java | 52 +++---------- .../blockmanagement/BlockPlacementPolicy.java | 77 +++++++++++++++++++ .../BlockPlacementPolicyDefault.java | 39 ++++++---- .../TestReplicationPolicy.java | 50 +++++++++++- 5 files changed, 167 insertions(+), 55 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7acb002ee3..a02ab0fb27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -93,6 +93,10 @@ Trunk (unreleased changes) HDFS-3478. Test quotas with Long.Max_Value. (Sujay Rau via eli) + HDFS-3498. Support replica removal in BlockPlacementPolicy and make + BlockPlacementPolicyDefault extensible for reusing code in subclasses. + (Junping Du via szetszwo) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 527a99765c..7e4a4857c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2259,30 +2259,14 @@ private void chooseExcessReplicates(Collection nonExcess, BlockCollection bc = getBlockCollection(b); final Map> rackMap = new HashMap>(); - for(final Iterator iter = nonExcess.iterator(); - iter.hasNext(); ) { - final DatanodeDescriptor node = iter.next(); - final String rackName = node.getNetworkLocation(); - List datanodeList = rackMap.get(rackName); - if (datanodeList == null) { - datanodeList = new ArrayList(); - rackMap.put(rackName, datanodeList); - } - datanodeList.add(node); - } + final List moreThanOne = new ArrayList(); + final List exactlyOne = new ArrayList(); // split nodes into two sets - // priSet contains nodes on rack with more than one replica - // remains contains the remaining nodes - final List priSet = new ArrayList(); - final List remains = new ArrayList(); - for(List datanodeList : rackMap.values()) { - if (datanodeList.size() == 1 ) { - remains.add(datanodeList.get(0)); - } else { - priSet.addAll(datanodeList); - } - } + // moreThanOne contains nodes on rack with more than one replica + // exactlyOne contains the remaining nodes + replicator.splitNodesWithRack(nonExcess, rackMap, moreThanOne, + exactlyOne); // pick one node to delete that favors the delete hint // otherwise pick one with least space from priSet if it is not empty @@ -2292,30 +2276,18 @@ private void chooseExcessReplicates(Collection nonExcess, // check if we can delete delNodeHint final DatanodeInfo cur; if (firstOne && delNodeHint !=null && nonExcess.contains(delNodeHint) - && (priSet.contains(delNodeHint) - || (addedNode != null && !priSet.contains(addedNode))) ) { + && (moreThanOne.contains(delNodeHint) + || (addedNode != null && !moreThanOne.contains(addedNode))) ) { cur = delNodeHint; } else { // regular excessive replica removal cur = replicator.chooseReplicaToDelete(bc, b, replication, - priSet, remains); + moreThanOne, exactlyOne); } firstOne = false; - // adjust rackmap, priSet, and remains - String rack = cur.getNetworkLocation(); - final List datanodes = rackMap.get(rack); - datanodes.remove(cur); - if (datanodes.isEmpty()) { - rackMap.remove(rack); - } - if (priSet.remove(cur)) { - if (datanodes.size() == 1) { - priSet.remove(datanodes.get(0)); - remains.add(datanodes.get(0)); - } - } else { - remains.remove(cur); - } + // adjust rackmap, moreThanOne, and exactlyOne + replicator.adjustSetsWithChosenReplica(rackMap, moreThanOne, + exactlyOne, cur); nonExcess.remove(cur); addToExcessReplicate(cur, b); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java index e1efae5419..e3317467bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java @@ -21,12 +21,14 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.namenode.FSClusterStats; import org.apache.hadoop.net.NetworkTopology; @@ -241,5 +243,80 @@ public DatanodeDescriptor[] chooseTarget(String srcPath, excludedNodes, blocksize); } + + /** + * Adjust rackmap, moreThanOne, and exactlyOne after removing replica on cur. + * + * @param rackMap a map from rack to replica + * @param moreThanOne The List of replica nodes on rack which has more than + * one replica + * @param exactlyOne The List of replica nodes on rack with only one replica + * @param cur current replica to remove + */ + public void adjustSetsWithChosenReplica(final Map> rackMap, + final List moreThanOne, + final List exactlyOne, final DatanodeInfo cur) { + + String rack = getRack(cur); + final List datanodes = rackMap.get(rack); + datanodes.remove(cur); + if (datanodes.isEmpty()) { + rackMap.remove(rack); + } + if (moreThanOne.remove(cur)) { + if (datanodes.size() == 1) { + moreThanOne.remove(datanodes.get(0)); + exactlyOne.add(datanodes.get(0)); + } + } else { + exactlyOne.remove(cur); + } + } + + /** + * Get rack string from a data node + * @param datanode + * @return rack of data node + */ + protected String getRack(final DatanodeInfo datanode) { + return datanode.getNetworkLocation(); + } + + /** + * Split data nodes into two sets, one set includes nodes on rack with + * more than one replica, the other set contains the remaining nodes. + * + * @param dataNodes + * @param rackMap a map from rack to datanodes + * @param moreThanOne contains nodes on rack with more than one replica + * @param exactlyOne remains contains the remaining nodes + */ + public void splitNodesWithRack( + Collection dataNodes, + final Map> rackMap, + final List moreThanOne, + final List exactlyOne) { + for(DatanodeDescriptor node : dataNodes) { + final String rackName = getRack(node); + List datanodeList = rackMap.get(rackName); + if (datanodeList == null) { + datanodeList = new ArrayList(); + rackMap.put(rackName, datanodeList); + } + datanodeList.add(node); + } + + // split nodes into two sets + for(List datanodeList : rackMap.values()) { + if (datanodeList.size() == 1) { + // exactlyOne contains nodes on rack with only one replica + exactlyOne.add(datanodeList.get(0)); + } else { + // moreThanOne contains nodes on rack with more than one replica + moreThanOne.addAll(datanodeList); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 6995a2e30f..350f863eae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -56,15 +56,15 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { "For more information, please enable DEBUG log level on " + ((Log4JLogger)LOG).getLogger().getName(); - private boolean considerLoad; + protected boolean considerLoad; private boolean preferLocalNode = true; - private NetworkTopology clusterMap; + protected NetworkTopology clusterMap; private FSClusterStats stats; - private long heartbeatInterval; // interval for DataNode heartbeats + protected long heartbeatInterval; // interval for DataNode heartbeats /** * A miss of that many heartbeats is tolerated for replica deletion policy. */ - private int tolerateHeartbeatMultiplier; + protected int tolerateHeartbeatMultiplier; BlockPlacementPolicyDefault(Configuration conf, FSClusterStats stats, NetworkTopology clusterMap) { @@ -88,7 +88,7 @@ public void initialize(Configuration conf, FSClusterStats stats, DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_DEFAULT); } - private ThreadLocal threadLocalBuilder = + protected ThreadLocal threadLocalBuilder = new ThreadLocal() { @Override protected StringBuilder initialValue() { @@ -229,7 +229,7 @@ private DatanodeDescriptor chooseTarget(int numOfReplicas, * choose a node on the same rack * @return the chosen node */ - private DatanodeDescriptor chooseLocalNode( + protected DatanodeDescriptor chooseLocalNode( DatanodeDescriptor localMachine, HashMap excludedNodes, long blocksize, @@ -263,7 +263,7 @@ private DatanodeDescriptor chooseLocalNode( * in the cluster. * @return the chosen node */ - private DatanodeDescriptor chooseLocalRack( + protected DatanodeDescriptor chooseLocalRack( DatanodeDescriptor localMachine, HashMap excludedNodes, long blocksize, @@ -316,7 +316,7 @@ private DatanodeDescriptor chooseLocalRack( * from the local rack */ - private void chooseRemoteRack(int numOfReplicas, + protected void chooseRemoteRack(int numOfReplicas, DatanodeDescriptor localMachine, HashMap excludedNodes, long blocksize, @@ -338,7 +338,7 @@ private void chooseRemoteRack(int numOfReplicas, /* Randomly choose one target from nodes. * @return the chosen node */ - private DatanodeDescriptor chooseRandom( + protected DatanodeDescriptor chooseRandom( String nodes, HashMap excludedNodes, long blocksize, @@ -382,7 +382,7 @@ private DatanodeDescriptor chooseRandom( /* Randomly choose numOfReplicas targets from nodes. */ - private void chooseRandom(int numOfReplicas, + protected void chooseRandom(int numOfReplicas, String nodes, HashMap excludedNodes, long blocksize, @@ -438,7 +438,7 @@ private boolean isGoodTarget(DatanodeDescriptor node, this.considerLoad, results); } - private boolean isGoodTarget(DatanodeDescriptor node, + protected boolean isGoodTarget(DatanodeDescriptor node, long blockSize, int maxTargetPerLoc, boolean considerLoad, List results) { @@ -574,8 +574,7 @@ public DatanodeDescriptor chooseReplicaToDelete(BlockCollection bc, // pick replica from the first Set. If first is empty, then pick replicas // from second set. - Iterator iter = - first.isEmpty() ? second.iterator() : first.iterator(); + Iterator iter = pickupReplicaSet(first, second); // Pick the node with the oldest heartbeat or with the least free space, // if all hearbeats are within the tolerable heartbeat interval @@ -594,6 +593,20 @@ public DatanodeDescriptor chooseReplicaToDelete(BlockCollection bc, } return oldestHeartbeatNode != null ? oldestHeartbeatNode : minSpaceNode; } + + /** + * Pick up replica node set for deleting replica as over-replicated. + * First set contains replica nodes on rack with more than one + * replica while second set contains remaining replica nodes. + * So pick up first set if not empty. If first is empty, then pick second. + */ + protected Iterator pickupReplicaSet( + Collection first, + Collection second) { + Iterator iter = + first.isEmpty() ? second.iterator() : first.iterator(); + return iter; + } @VisibleForTesting void setPreferLocalNode(boolean prefer) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index ce570f7eba..aefd0befed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -24,6 +24,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Random; import org.apache.hadoop.conf.Configuration; @@ -34,7 +35,6 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.net.NetworkTopology; @@ -61,7 +61,7 @@ public static void setupCluster() throws Exception { DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/d1/r2"), DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/d1/r2"), DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/d2/r3"), - DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/d2/r3") + DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/d2/r3") }; FileSystem.setDefaultUri(conf, "hdfs://localhost:0"); @@ -587,4 +587,50 @@ private void assertTheChosenBlocks( fifthPrioritySize, chosenBlocks.get( UnderReplicatedBlocks.QUEUE_WITH_CORRUPT_BLOCKS).size()); } + + /** + * Test for the chooseReplicaToDelete are processed based on + * block locality and free space + */ + @Test + public void testChooseReplicaToDelete() throws Exception { + List replicaNodeList = new + ArrayList(); + final Map> rackMap + = new HashMap>(); + + dataNodes[0].setRemaining(4*1024*1024); + replicaNodeList.add(dataNodes[0]); + + dataNodes[1].setRemaining(3*1024*1024); + replicaNodeList.add(dataNodes[1]); + + dataNodes[2].setRemaining(2*1024*1024); + replicaNodeList.add(dataNodes[2]); + + dataNodes[5].setRemaining(1*1024*1024); + replicaNodeList.add(dataNodes[5]); + + List first = new ArrayList(); + List second = new ArrayList(); + replicator.splitNodesWithRack( + replicaNodeList, rackMap, first, second); + // dataNodes[0] and dataNodes[1] are in first set as their rack has two + // replica nodes, while datanodes[2] and dataNodes[5] are in second set. + assertEquals(2, first.size()); + assertEquals(2, second.size()); + DatanodeDescriptor chosenNode = replicator.chooseReplicaToDelete( + null, null, (short)3, first, second); + // Within first set, dataNodes[1] with less free space + assertEquals(chosenNode, dataNodes[1]); + + replicator.adjustSetsWithChosenReplica( + rackMap, first, second, chosenNode); + assertEquals(0, first.size()); + assertEquals(3, second.size()); + // Within second set, dataNodes[5] with less free space + chosenNode = replicator.chooseReplicaToDelete( + null, null, (short)2, first, second); + assertEquals(chosenNode, dataNodes[5]); + } } From 06237f6f4c5319d3281e71484db4c0c0d4da98e4 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Tue, 26 Jun 2012 16:23:00 +0000 Subject: [PATCH 79/91] HADOOP-8129. ViewFileSystemTestSetup setupForViewFileSystem is erring (Ahmed Radwan and Ravi Prakash via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1354093 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop/fs/viewfs/ViewFileSystemTestSetup.java | 14 +++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 1fff076215..1fed42d0cb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -731,6 +731,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8495. Update Netty to avoid leaking file descriptors during shuffle (Jason Lowe via tgraves) + HADOOP-8129. ViewFileSystemTestSetup setupForViewFileSystem is erring + (Ahmed Radwan and Ravi Prakash via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java index 11f4d7af71..525f28bea7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java @@ -51,7 +51,19 @@ static public FileSystem setupForViewFileSystem(Configuration conf, FileSystem f /** * create the test root on local_fs - the mount table will point here */ - fsTarget.mkdirs(FileSystemTestHelper.getTestRootPath(fsTarget)); + Path targetOfTests = FileSystemTestHelper.getTestRootPath(fsTarget); + // In case previous test was killed before cleanup + fsTarget.delete(targetOfTests, true); + fsTarget.mkdirs(targetOfTests); + + // Setup a link from viewfs to targetfs for the first component of + // path of testdir. + String testDir = FileSystemTestHelper.getTestRootPath(fsTarget).toUri() + .getPath(); + int indexOf2ndSlash = testDir.indexOf('/', 1); + String testDirFirstComponent = testDir.substring(0, indexOf2ndSlash); + ConfigUtil.addLink(conf, testDirFirstComponent, fsTarget.makeQualified( + new Path(testDirFirstComponent)).toUri()); // viewFs://home => fsTarget://home String homeDirRoot = fsTarget.getHomeDirectory() From cda64a6812b2ea69ccbbfc3e5261175c680fd7e0 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Tue, 26 Jun 2012 18:14:09 +0000 Subject: [PATCH 80/91] HDFS-3535. Audit logging should log denied accesses. Contributed by Andy Isaacson git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1354144 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hdfs/server/namenode/FSNamesystem.java | 228 +++++++++++++++++- .../hdfs/server/namenode/TestAuditLogs.java | 162 +++++++++++++ .../hadoop/hdfs/server/namenode/TestFsck.java | 3 +- 4 files changed, 381 insertions(+), 14 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a02ab0fb27..ee76afad94 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -244,6 +244,8 @@ Branch-2 ( Unreleased changes ) HDFS-3516. Check content-type in WebHdfsFileSystem. (szetszwo) + HDFS-3535. Audit logging should log denied accesses. (Andy Isaacson via eli) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index f965ed04d2..9531ee2adf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -240,8 +240,15 @@ protected StringBuilder initialValue() { private static final void logAuditEvent(UserGroupInformation ugi, InetAddress addr, String cmd, String src, String dst, HdfsFileStatus stat) { + logAuditEvent(true, ugi, addr, cmd, src, dst, stat); + } + + private static final void logAuditEvent(boolean succeeded, + UserGroupInformation ugi, InetAddress addr, String cmd, String src, + String dst, HdfsFileStatus stat) { final StringBuilder sb = auditBuffer.get(); sb.setLength(0); + sb.append("allowed=").append(succeeded).append("\t"); sb.append("ugi=").append(ugi).append("\t"); sb.append("ip=").append(addr).append("\t"); sb.append("cmd=").append(cmd).append("\t"); @@ -1018,6 +1025,21 @@ private boolean isAccessTimeSupported() { void setPermission(String src, FsPermission permission) throws AccessControlException, FileNotFoundException, SafeModeException, UnresolvedLinkException, IOException { + try { + setPermissionInt(src, permission); + } catch (AccessControlException e) { + if (auditLog.isInfoEnabled() && isExternalInvocation()) { + logAuditEvent(false, UserGroupInformation.getCurrentUser(), + Server.getRemoteIp(), + "setPermission", src, null, null); + } + throw e; + } + } + + private void setPermissionInt(String src, FsPermission permission) + throws AccessControlException, FileNotFoundException, SafeModeException, + UnresolvedLinkException, IOException { HdfsFileStatus resultingStat = null; writeLock(); try { @@ -1049,6 +1071,21 @@ void setPermission(String src, FsPermission permission) void setOwner(String src, String username, String group) throws AccessControlException, FileNotFoundException, SafeModeException, UnresolvedLinkException, IOException { + try { + setOwnerInt(src, username, group); + } catch (AccessControlException e) { + if (auditLog.isInfoEnabled() && isExternalInvocation()) { + logAuditEvent(false, UserGroupInformation.getCurrentUser(), + Server.getRemoteIp(), + "setOwner", src, null, null); + } + throw e; + } + } + + private void setOwnerInt(String src, String username, String group) + throws AccessControlException, FileNotFoundException, SafeModeException, + UnresolvedLinkException, IOException { HdfsFileStatus resultingStat = null; writeLock(); try { @@ -1106,6 +1143,22 @@ LocatedBlocks getBlockLocations(String clientMachine, String src, LocatedBlocks getBlockLocations(String src, long offset, long length, boolean doAccessTime, boolean needBlockToken, boolean checkSafeMode) throws FileNotFoundException, UnresolvedLinkException, IOException { + try { + return getBlockLocationsInt(src, offset, length, doAccessTime, + needBlockToken, checkSafeMode); + } catch (AccessControlException e) { + if (auditLog.isInfoEnabled() && isExternalInvocation()) { + logAuditEvent(false, UserGroupInformation.getCurrentUser(), + Server.getRemoteIp(), + "open", src, null, null); + } + throw e; + } + } + + private LocatedBlocks getBlockLocationsInt(String src, long offset, long length, + boolean doAccessTime, boolean needBlockToken, boolean checkSafeMode) + throws FileNotFoundException, UnresolvedLinkException, IOException { if (isPermissionEnabled) { checkPathAccess(src, FsAction.READ); } @@ -1202,6 +1255,20 @@ private LocatedBlocks getBlockLocationsUpdateTimes(String src, */ void concat(String target, String [] srcs) throws IOException, UnresolvedLinkException { + try { + concatInt(target, srcs); + } catch (AccessControlException e) { + if (auditLog.isInfoEnabled() && isExternalInvocation()) { + logAuditEvent(false, UserGroupInformation.getLoginUser(), + Server.getRemoteIp(), + "concat", Arrays.toString(srcs), target, null); + } + throw e; + } + } + + private void concatInt(String target, String [] srcs) + throws IOException, UnresolvedLinkException { if(FSNamesystem.LOG.isDebugEnabled()) { FSNamesystem.LOG.debug("concat " + Arrays.toString(srcs) + " to " + target); @@ -1354,6 +1421,20 @@ private void concatInternal(String target, String [] srcs) * written to the edits log but is not flushed. */ void setTimes(String src, long mtime, long atime) + throws IOException, UnresolvedLinkException { + try { + setTimesInt(src, mtime, atime); + } catch (AccessControlException e) { + if (auditLog.isInfoEnabled() && isExternalInvocation()) { + logAuditEvent(false, UserGroupInformation.getCurrentUser(), + Server.getRemoteIp(), + "setTimes", src, null, null); + } + throw e; + } + } + + private void setTimesInt(String src, long mtime, long atime) throws IOException, UnresolvedLinkException { if (!isAccessTimeSupported() && atime != -1) { throw new IOException("Access time for hdfs is not configured. " + @@ -1390,6 +1471,21 @@ void setTimes(String src, long mtime, long atime) void createSymlink(String target, String link, PermissionStatus dirPerms, boolean createParent) throws IOException, UnresolvedLinkException { + try { + createSymlinkInt(target, link, dirPerms, createParent); + } catch (AccessControlException e) { + if (auditLog.isInfoEnabled() && isExternalInvocation()) { + logAuditEvent(false, UserGroupInformation.getCurrentUser(), + Server.getRemoteIp(), + "createSymlink", link, target, null); + } + throw e; + } + } + + private void createSymlinkInt(String target, String link, + PermissionStatus dirPerms, boolean createParent) + throws IOException, UnresolvedLinkException { HdfsFileStatus resultingStat = null; writeLock(); try { @@ -1457,8 +1553,22 @@ private void createSymlinkInternal(String target, String link, * @return true if successful; * false if file does not exist or is a directory */ - boolean setReplication(final String src, final short replication - ) throws IOException { + boolean setReplication(final String src, final short replication) + throws IOException { + try { + return setReplicationInt(src, replication); + } catch (AccessControlException e) { + if (auditLog.isInfoEnabled() && isExternalInvocation()) { + logAuditEvent(false, UserGroupInformation.getCurrentUser(), + Server.getRemoteIp(), + "setReplication", src, null, null); + } + throw e; + } + } + + private boolean setReplicationInt(final String src, final short replication) + throws IOException { blockManager.verifyReplication(src, replication, null); final boolean isFile; @@ -1491,7 +1601,7 @@ boolean setReplication(final String src, final short replication } return isFile; } - + long getPreferredBlockSize(String filename) throws IOException, UnresolvedLinkException { readLock(); @@ -1537,6 +1647,24 @@ void startFile(String src, PermissionStatus permissions, String holder, short replication, long blockSize) throws AccessControlException, SafeModeException, FileAlreadyExistsException, UnresolvedLinkException, FileNotFoundException, ParentNotDirectoryException, IOException { + try { + startFileInt(src, permissions, holder, clientMachine, flag, createParent, + replication, blockSize); + } catch (AccessControlException e) { + if (auditLog.isInfoEnabled() && isExternalInvocation()) { + logAuditEvent(false, UserGroupInformation.getCurrentUser(), + Server.getRemoteIp(), + "create", src, null, null); + } + throw e; + } + } + + private void startFileInt(String src, PermissionStatus permissions, String holder, + String clientMachine, EnumSet flag, boolean createParent, + short replication, long blockSize) throws AccessControlException, + SafeModeException, FileAlreadyExistsException, UnresolvedLinkException, + FileNotFoundException, ParentNotDirectoryException, IOException { writeLock(); try { checkOperation(OperationCategory.WRITE); @@ -1840,6 +1968,22 @@ LocatedBlock appendFile(String src, String holder, String clientMachine) throws AccessControlException, SafeModeException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, IOException { + try { + return appendFileInt(src, holder, clientMachine); + } catch (AccessControlException e) { + if (auditLog.isInfoEnabled() && isExternalInvocation()) { + logAuditEvent(false, UserGroupInformation.getCurrentUser(), + Server.getRemoteIp(), + "append", src, null, null); + } + throw e; + } + } + + private LocatedBlock appendFileInt(String src, String holder, String clientMachine) + throws AccessControlException, SafeModeException, + FileAlreadyExistsException, FileNotFoundException, + ParentNotDirectoryException, IOException { if (!supportAppends) { throw new UnsupportedOperationException( "Append is not enabled on this NameNode. Use the " + @@ -2326,6 +2470,20 @@ boolean checkFileProgress(INodeFile v, boolean checkall) { */ @Deprecated boolean renameTo(String src, String dst) + throws IOException, UnresolvedLinkException { + try { + return renameToInt(src, dst); + } catch (AccessControlException e) { + if (auditLog.isInfoEnabled() && isExternalInvocation()) { + logAuditEvent(false, UserGroupInformation.getCurrentUser(), + Server.getRemoteIp(), + "rename", src, dst, null); + } + throw e; + } + } + + private boolean renameToInt(String src, String dst) throws IOException, UnresolvedLinkException { boolean status = false; HdfsFileStatus resultingStat = null; @@ -2437,20 +2595,35 @@ private void renameToInternal(String src, String dst, * @see ClientProtocol#delete(String, boolean) for detailed descriptoin and * description of exceptions */ - boolean delete(String src, boolean recursive) - throws AccessControlException, SafeModeException, - UnresolvedLinkException, IOException { - if (NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("DIR* NameSystem.delete: " + src); - } - boolean status = deleteInternal(src, recursive, true); - if (status && auditLog.isInfoEnabled() && isExternalInvocation()) { - logAuditEvent(UserGroupInformation.getCurrentUser(), + boolean delete(String src, boolean recursive) + throws AccessControlException, SafeModeException, + UnresolvedLinkException, IOException { + try { + return deleteInt(src, recursive); + } catch (AccessControlException e) { + if (auditLog.isInfoEnabled() && isExternalInvocation()) { + logAuditEvent(false, UserGroupInformation.getCurrentUser(), Server.getRemoteIp(), "delete", src, null, null); } - return status; + throw e; + } + } + + private boolean deleteInt(String src, boolean recursive) + throws AccessControlException, SafeModeException, + UnresolvedLinkException, IOException { + if (NameNode.stateChangeLog.isDebugEnabled()) { + NameNode.stateChangeLog.debug("DIR* NameSystem.delete: " + src); } + boolean status = deleteInternal(src, recursive, true); + if (status && auditLog.isInfoEnabled() && isExternalInvocation()) { + logAuditEvent(UserGroupInformation.getCurrentUser(), + Server.getRemoteIp(), + "delete", src, null, null); + } + return status; + } /** * Remove a file/directory from the namespace. @@ -2606,6 +2779,20 @@ HdfsFileStatus getFileInfo(String src, boolean resolveLink) */ boolean mkdirs(String src, PermissionStatus permissions, boolean createParent) throws IOException, UnresolvedLinkException { + try { + return mkdirsInt(src, permissions, createParent); + } catch (AccessControlException e) { + if (auditLog.isInfoEnabled() && isExternalInvocation()) { + logAuditEvent(false, UserGroupInformation.getCurrentUser(), + Server.getRemoteIp(), + "mkdirs", src, null, null); + } + throw e; + } + } + + private boolean mkdirsInt(String src, PermissionStatus permissions, + boolean createParent) throws IOException, UnresolvedLinkException { boolean status = false; if(NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug("DIR* NameSystem.mkdirs: " + src); @@ -3057,6 +3244,21 @@ void renewLease(String holder) throws IOException { */ DirectoryListing getListing(String src, byte[] startAfter, boolean needLocation) + throws AccessControlException, UnresolvedLinkException, IOException { + try { + return getListingInt(src, startAfter, needLocation); + } catch (AccessControlException e) { + if (auditLog.isInfoEnabled() && isExternalInvocation()) { + logAuditEvent(false, UserGroupInformation.getCurrentUser(), + Server.getRemoteIp(), + "listStatus", src, null, null); + } + throw e; + } + } + + private DirectoryListing getListingInt(String src, byte[] startAfter, + boolean needLocation) throws AccessControlException, UnresolvedLinkException, IOException { DirectoryListing dl; readLock(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java new file mode 100644 index 0000000000..694d84f4d1 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.namenode; + +import static org.junit.Assert.*; +import org.junit.Before; +import org.junit.After; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.util.regex.Pattern; + +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; +import org.apache.log4j.RollingFileAppender; +import org.junit.Test; + +/** + * A JUnit test that audit logs are generated + */ +public class TestAuditLogs { + static final String auditLogFile = System.getProperty("test.build.dir", + "build/test") + "/audit.log"; + + // Pattern for: + // allowed=(true|false) ugi=name ip=/address cmd={cmd} src={path} dst=null perm=null + static final Pattern auditPattern = Pattern.compile( + "allowed=.*?\\s" + + "ugi=.*?\\s" + + "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + + "cmd=.*?\\ssrc=.*?\\sdst=null\\s" + + "perm=.*?"); + static final Pattern successPattern = Pattern.compile( + ".*allowed=true.*"); + static final String username = "bob"; + static final String[] groups = { "group1" }; + static final String fileName = "/srcdat"; + + DFSTestUtil util; + MiniDFSCluster cluster; + FileSystem fs; + String fnames[]; + Configuration conf; + UserGroupInformation userGroupInfo; + + @Before + public void setupCluster() throws Exception { + conf = new HdfsConfiguration(); + final long precision = 1L; + conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, precision); + conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); + util = new DFSTestUtil("TestAuditAllowed", 20, 3, 8*1024); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + fs = cluster.getFileSystem(); + util.createFiles(fs, fileName); + + fnames = util.getFileNames(fileName); + util.waitReplication(fs, fileName, (short)3); + userGroupInfo = UserGroupInformation.createUserForTesting(username, groups); + } + + @After + public void teardownCluster() throws Exception { + util.cleanup(fs, "/srcdat"); + fs.close(); + cluster.shutdown(); + } + + /** test that allowed operation puts proper entry in audit log */ + @Test + public void testAuditAllowed() throws Exception { + final Path file = new Path(fnames[0]); + FileSystem userfs = DFSTestUtil.getFileSystemAs(userGroupInfo, conf); + + setupAuditLogs(); + InputStream istream = userfs.open(file); + int val = istream.read(); + istream.close(); + verifyAuditLogs(true); + assertTrue("failed to read from file", val > 0); + } + + /** test that denied operation puts proper entry in audit log */ + @Test + public void testAuditDenied() throws Exception { + final Path file = new Path(fnames[0]); + FileSystem userfs = DFSTestUtil.getFileSystemAs(userGroupInfo, conf); + + fs.setPermission(file, new FsPermission((short)0600)); + fs.setOwner(file, "root", null); + + setupAuditLogs(); + + try { + userfs.open(file); + fail("open must not succeed"); + } catch(AccessControlException e) { + System.out.println("got access denied, as expected."); + } + verifyAuditLogs(false); + } + + /** Sets up log4j logger for auditlogs */ + private void setupAuditLogs() throws IOException { + File file = new File(auditLogFile); + if (file.exists()) { + file.delete(); + } + Logger logger = ((Log4JLogger) FSNamesystem.auditLog).getLogger(); + logger.setLevel(Level.INFO); + PatternLayout layout = new PatternLayout("%m%n"); + RollingFileAppender appender = new RollingFileAppender(layout, auditLogFile); + logger.addAppender(appender); + } + + private void verifyAuditLogs(boolean expectSuccess) throws IOException { + // Turn off the logs + Logger logger = ((Log4JLogger) FSNamesystem.auditLog).getLogger(); + logger.setLevel(Level.OFF); + + // Ensure audit log has only one entry + BufferedReader reader = new BufferedReader(new FileReader(auditLogFile)); + String line = reader.readLine(); + assertNotNull(line); + assertTrue("Expected audit event not found in audit log", + auditPattern.matcher(line).matches()); + assertTrue("Expected success=" + expectSuccess, + successPattern.matcher(line).matches() == expectSuccess); + assertNull("Unexpected event in audit log", reader.readLine()); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 9b59802b1d..531dc879d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -76,8 +76,9 @@ public class TestFsck { "build/test") + "/audit.log"; // Pattern for: - // ugi=name ip=/address cmd=FSCK src=/ dst=null perm=null + // allowed=true ugi=name ip=/address cmd=FSCK src=/ dst=null perm=null static final Pattern fsckPattern = Pattern.compile( + "allowed=.*?\\s" + "ugi=.*?\\s" + "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + "cmd=fsck\\ssrc=\\/\\sdst=null\\s" + From 51d7c7685d7f5bf0afa4e3796b72b126f9185869 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Tue, 26 Jun 2012 19:17:33 +0000 Subject: [PATCH 81/91] MAPREDUCE-4228. mapreduce.job.reduce.slowstart.completedmaps is not working properly (Jason Lowe via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1354181 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../v2/app/rm/RMContainerAllocator.java | 21 +++--- .../v2/app/TestRMContainerAllocator.java | 73 ++++++++++++------- 3 files changed, 61 insertions(+), 36 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 5cce59ba04..36dc62f5f5 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -619,6 +619,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4295. RM crashes due to DNS issue (tgraves) + MAPREDUCE-4228. mapreduce.job.reduce.slowstart.completedmaps is not working + properly (Jason Lowe via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index efef456f10..a0ba0e4c69 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -417,15 +417,6 @@ public void scheduleReduces( LOG.info("Recalculating schedule..."); - //if all maps are assigned, then ramp up all reduces irrespective of the - //headroom - if (scheduledMaps == 0 && numPendingReduces > 0) { - LOG.info("All maps assigned. " + - "Ramping up all remaining reduces:" + numPendingReduces); - scheduleAllReduces(); - return; - } - //check for slow start if (!getIsReduceStarted()) {//not set yet int completedMapsForReduceSlowstart = (int)Math.ceil(reduceSlowStart * @@ -441,6 +432,15 @@ public void scheduleReduces( } } + //if all maps are assigned, then ramp up all reduces irrespective of the + //headroom + if (scheduledMaps == 0 && numPendingReduces > 0) { + LOG.info("All maps assigned. " + + "Ramping up all remaining reduces:" + numPendingReduces); + scheduleAllReduces(); + return; + } + float completedMapPercent = 0f; if (totalMaps != 0) {//support for 0 maps completedMapPercent = (float)completedMaps/totalMaps; @@ -498,7 +498,8 @@ public void scheduleReduces( } } - private void scheduleAllReduces() { + @Private + public void scheduleAllReduces() { for (ContainerRequest req : pendingReduces) { scheduledRequests.addReduce(req); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java index 98bc020ecb..33306f4593 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java @@ -18,15 +18,24 @@ package org.apache.hadoop.mapreduce.v2.app; +import static org.mockito.Matchers.anyFloat; +import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.isA; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import junit.framework.Assert; @@ -65,9 +74,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.Event; @@ -76,13 +86,11 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; -import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.After; import org.junit.Test; @@ -428,29 +436,21 @@ protected ContainerAllocator createContainerAllocator( // Finish off 1 map. Iterator it = job.getTasks().values().iterator(); - finishNextNTasks(mrApp, it, 1); + finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 1); allocator.schedule(); rmDispatcher.await(); Assert.assertEquals(0.095f, job.getProgress(), 0.001f); Assert.assertEquals(0.095f, rmApp.getProgress(), 0.001f); // Finish off 7 more so that map-progress is 80% - finishNextNTasks(mrApp, it, 7); + finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 7); allocator.schedule(); rmDispatcher.await(); Assert.assertEquals(0.41f, job.getProgress(), 0.001f); Assert.assertEquals(0.41f, rmApp.getProgress(), 0.001f); // Finish off the 2 remaining maps - finishNextNTasks(mrApp, it, 2); - - // Wait till all reduce-attempts request for containers - for (Task t : job.getTasks().values()) { - if (t.getType() == TaskType.REDUCE) { - mrApp.waitForState(t.getAttempts().values().iterator().next(), - TaskAttemptState.UNASSIGNED); - } - } + finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 2); allocator.schedule(); rmDispatcher.await(); @@ -467,7 +467,7 @@ protected ContainerAllocator createContainerAllocator( } // Finish off 2 reduces - finishNextNTasks(mrApp, it, 2); + finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 2); allocator.schedule(); rmDispatcher.await(); @@ -475,7 +475,7 @@ protected ContainerAllocator createContainerAllocator( Assert.assertEquals(0.59f, rmApp.getProgress(), 0.001f); // Finish off the remaining 8 reduces. - finishNextNTasks(mrApp, it, 8); + finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 8); allocator.schedule(); rmDispatcher.await(); // Remaining is JobCleanup @@ -483,19 +483,28 @@ protected ContainerAllocator createContainerAllocator( Assert.assertEquals(0.95f, rmApp.getProgress(), 0.001f); } - private void finishNextNTasks(MRApp mrApp, Iterator it, int nextN) - throws Exception { + private void finishNextNTasks(DrainDispatcher rmDispatcher, MockNM node, + MRApp mrApp, Iterator it, int nextN) throws Exception { Task task; for (int i=0; i contStatus = new ArrayList(1); + contStatus.add(BuilderUtils.newContainerStatus(attempt.getAssignedContainerID(), + ContainerState.COMPLETE, "", 0)); + Map> statusUpdate = + new HashMap>(1); + statusUpdate.put(mrApp.getAppID(), contStatus); + node.nodeHeartbeat(statusUpdate, true); + rmDispatcher.await(); mrApp.getContext().getEventHandler().handle( - new TaskAttemptEvent(attempt.getID(), TaskAttemptEventType.TA_DONE)); + new TaskAttemptEvent(attempt.getID(), TaskAttemptEventType.TA_DONE)); mrApp.waitForState(task, TaskState.SUCCEEDED); } @@ -576,21 +585,21 @@ protected ContainerAllocator createContainerAllocator( Iterator it = job.getTasks().values().iterator(); // Finish off 1 map so that map-progress is 10% - finishNextNTasks(mrApp, it, 1); + finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 1); allocator.schedule(); rmDispatcher.await(); Assert.assertEquals(0.14f, job.getProgress(), 0.001f); Assert.assertEquals(0.14f, rmApp.getProgress(), 0.001f); // Finish off 5 more map so that map-progress is 60% - finishNextNTasks(mrApp, it, 5); + finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 5); allocator.schedule(); rmDispatcher.await(); Assert.assertEquals(0.59f, job.getProgress(), 0.001f); Assert.assertEquals(0.59f, rmApp.getProgress(), 0.001f); // Finish off remaining map so that map-progress is 100% - finishNextNTasks(mrApp, it, 4); + finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 4); allocator.schedule(); rmDispatcher.await(); Assert.assertEquals(0.95f, job.getProgress(), 0.001f); @@ -1338,6 +1347,18 @@ public void testReduceScheduling() throws Exception { maxReduceRampupLimit, reduceSlowStart); verify(allocator, never()).setIsReduceStarted(true); + // verify slow-start still in effect when no more maps need to + // be scheduled but some have yet to complete + allocator.scheduleReduces( + totalMaps, succeededMaps, + 0, scheduledReduces, + totalMaps - succeededMaps, assignedReduces, + mapResourceReqt, reduceResourceReqt, + numPendingReduces, + maxReduceRampupLimit, reduceSlowStart); + verify(allocator, never()).setIsReduceStarted(true); + verify(allocator, never()).scheduleAllReduces(); + succeededMaps = 3; allocator.scheduleReduces( totalMaps, succeededMaps, From 4b48b25ab8c4c3270b469e43b865ef66a303dadf Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Wed, 27 Jun 2012 02:51:39 +0000 Subject: [PATCH 82/91] HDFS-3551. WebHDFS CREATE should use client location for HTTP redirection. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1354316 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../blockmanagement/DatanodeManager.java | 4 + .../server/blockmanagement/Host2NodesMap.java | 12 ++ .../web/resources/NamenodeWebHdfsMethods.java | 45 ++++-- .../resources/TestWebHdfsDataLocality.java | 140 ++++++++++++++++++ .../hadoop/hdfs/web/WebHdfsTestUtil.java | 6 + 6 files changed, 198 insertions(+), 12 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ee76afad94..bf58013749 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -381,6 +381,9 @@ Branch-2 ( Unreleased changes ) HDFS-3428. Move DelegationTokenRenewer to common (tucu) + HDFS-3551. WebHDFS CREATE should use client location for HTTP redirection. + (szetszwo) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index c58b857ebf..474feb5247 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -1053,4 +1053,8 @@ public void clearPendingQueues() { } } + @Override + public String toString() { + return getClass().getSimpleName() + ": " + host2DatanodeMap; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java index 68ea1f1710..082816d0e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java @@ -17,7 +17,9 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import java.util.Arrays; import java.util.HashMap; +import java.util.Map; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -156,4 +158,14 @@ DatanodeDescriptor getDatanodeByHost(String ipAddr) { hostmapLock.readLock().unlock(); } } + + @Override + public String toString() { + final StringBuilder b = new StringBuilder(getClass().getSimpleName()) + .append("["); + for(Map.Entry e : map.entrySet()) { + b.append("\n " + e.getKey() + " => " + Arrays.asList(e.getValue())); + } + return b.append("\n]").toString(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index de8f256705..37781ea120 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.namenode.NameNode; @@ -115,6 +116,11 @@ public static String getRemoteAddress() { return REMOTE_ADDRESS.get(); } + /** Set the remote client address. */ + static void setRemoteAddress(String remoteAddress) { + REMOTE_ADDRESS.set(remoteAddress); + } + private @Context ServletContext context; private @Context HttpServletRequest request; private @Context HttpServletResponse response; @@ -134,12 +140,26 @@ private void init(final UserGroupInformation ugi, response.setContentType(null); } - private static DatanodeInfo chooseDatanode(final NameNode namenode, + static DatanodeInfo chooseDatanode(final NameNode namenode, final String path, final HttpOpParam.Op op, final long openOffset, - Configuration conf) throws IOException { - if (op == GetOpParam.Op.OPEN + final long blocksize, Configuration conf) throws IOException { + final BlockManager bm = namenode.getNamesystem().getBlockManager(); + + if (op == PutOpParam.Op.CREATE) { + //choose a datanode near to client + final DatanodeDescriptor clientNode = bm.getDatanodeManager( + ).getDatanodeByHost(getRemoteAddress()); + if (clientNode != null) { + final DatanodeDescriptor[] datanodes = bm.getBlockPlacementPolicy( + ).chooseTarget(path, 1, clientNode, null, blocksize); + if (datanodes.length > 0) { + return datanodes[0]; + } + } + } else if (op == GetOpParam.Op.OPEN || op == GetOpParam.Op.GETFILECHECKSUM || op == PostOpParam.Op.APPEND) { + //choose a datanode containing a replica final NamenodeProtocols np = namenode.getRpcServer(); final HdfsFileStatus status = np.getFileInfo(path); if (status == null) { @@ -158,14 +178,13 @@ private static DatanodeInfo chooseDatanode(final NameNode namenode, final LocatedBlocks locations = np.getBlockLocations(path, offset, 1); final int count = locations.locatedBlockCount(); if (count > 0) { - return JspHelper.bestNode(locations.get(0), conf); + return JspHelper.bestNode(locations.get(0).getLocations(), false, conf); } } } - return (DatanodeDescriptor)namenode.getNamesystem().getBlockManager( - ).getDatanodeManager().getNetworkTopology().chooseRandom( - NodeBase.ROOT); + return (DatanodeDescriptor)bm.getDatanodeManager().getNetworkTopology( + ).chooseRandom(NodeBase.ROOT); } private Token generateDelegationToken( @@ -183,9 +202,11 @@ private URI redirectURI(final NameNode namenode, final UserGroupInformation ugi, final DelegationParam delegation, final UserParam username, final DoAsParam doAsUser, final String path, final HttpOpParam.Op op, final long openOffset, + final long blocksize, final Param... parameters) throws URISyntaxException, IOException { final Configuration conf = (Configuration)context.getAttribute(JspHelper.CURRENT_CONF); - final DatanodeInfo dn = chooseDatanode(namenode, path, op, openOffset, conf); + final DatanodeInfo dn = chooseDatanode(namenode, path, op, openOffset, + blocksize, conf); final String delegationQuery; if (!UserGroupInformation.isSecurityEnabled()) { @@ -356,7 +377,7 @@ private Response put( case CREATE: { final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser, - fullpath, op.getValue(), -1L, + fullpath, op.getValue(), -1L, blockSize.getValue(conf), permission, overwrite, bufferSize, replication, blockSize); return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build(); } @@ -502,7 +523,7 @@ private Response post( case APPEND: { final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser, - fullpath, op.getValue(), -1L, bufferSize); + fullpath, op.getValue(), -1L, -1L, bufferSize); return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build(); } default: @@ -598,7 +619,7 @@ private Response get( case OPEN: { final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser, - fullpath, op.getValue(), offset.getValue(), offset, length, bufferSize); + fullpath, op.getValue(), offset.getValue(), -1L, offset, length, bufferSize); return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build(); } case GET_BLOCK_LOCATIONS: @@ -634,7 +655,7 @@ private Response get( case GETFILECHECKSUM: { final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser, - fullpath, op.getValue(), -1L); + fullpath, op.getValue(), -1L, -1L); return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build(); } case GETDELEGATIONTOKEN: diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java new file mode 100644 index 0000000000..74373be6e5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.web.resources; + +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.LeaseManager; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.hdfs.web.WebHdfsTestUtil; +import org.apache.hadoop.hdfs.web.resources.GetOpParam; +import org.apache.hadoop.hdfs.web.resources.PostOpParam; +import org.apache.hadoop.hdfs.web.resources.PutOpParam; +import org.apache.log4j.Level; +import org.junit.Assert; +import org.junit.Test; + +/** + * Test WebHDFS which provides data locality using HTTP redirection. + */ +public class TestWebHdfsDataLocality { + static final Log LOG = LogFactory.getLog(TestWebHdfsDataLocality.class); + { + ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.OFF); + ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.OFF); + ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.OFF); + } + + private static final String RACK0 = "/rack0"; + private static final String RACK1 = "/rack1"; + private static final String RACK2 = "/rack2"; + + @Test + public void testDataLocality() throws Exception { + final Configuration conf = WebHdfsTestUtil.createConf(); + final String[] racks = {RACK0, RACK0, RACK1, RACK1, RACK2, RACK2}; + final int nDataNodes = racks.length; + LOG.info("nDataNodes=" + nDataNodes + ", racks=" + Arrays.asList(racks)); + + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(nDataNodes) + .racks(racks) + .build(); + try { + cluster.waitActive(); + + final DistributedFileSystem dfs = cluster.getFileSystem(); + final NameNode namenode = cluster.getNameNode(); + final DatanodeManager dm = namenode.getNamesystem().getBlockManager( + ).getDatanodeManager(); + LOG.info("dm=" + dm); + + final long blocksize = DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT; + final String f = "/foo"; + + { //test CREATE + for(int i = 0; i < nDataNodes; i++) { + //set client address to a particular datanode + final DataNode dn = cluster.getDataNodes().get(i); + final String ipAddr = dm.getDatanode(dn.getDatanodeId()).getIpAddr(); + NamenodeWebHdfsMethods.setRemoteAddress(ipAddr); + + //The chosen datanode must be the same as the client address + final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( + namenode, f, PutOpParam.Op.CREATE, -1L, blocksize, conf); + Assert.assertEquals(ipAddr, chosen.getIpAddr()); + } + } + + //create a file with one replica. + final Path p = new Path(f); + final FSDataOutputStream out = dfs.create(p, (short)1); + out.write(1); + out.close(); + + //get replica location. + final LocatedBlocks locatedblocks = NameNodeAdapter.getBlockLocations( + namenode, f, 0, 1); + final List lb = locatedblocks.getLocatedBlocks(); + Assert.assertEquals(1, lb.size()); + final DatanodeInfo[] locations = lb.get(0).getLocations(); + Assert.assertEquals(1, locations.length); + final DatanodeInfo expected = locations[0]; + + //For GETFILECHECKSUM, OPEN and APPEND, + //the chosen datanode must be the same as the replica location. + + { //test GETFILECHECKSUM + final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( + namenode, f, GetOpParam.Op.GETFILECHECKSUM, -1L, blocksize, conf); + Assert.assertEquals(expected, chosen); + } + + { //test OPEN + final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( + namenode, f, GetOpParam.Op.OPEN, 0, blocksize, conf); + Assert.assertEquals(expected, chosen); + } + + { //test APPEND + final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( + namenode, f, PostOpParam.Op.APPEND, -1L, blocksize, conf); + Assert.assertEquals(expected, chosen); + } + } finally { + cluster.shutdown(); + } + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java index 38e2168519..9ae0fb28c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java @@ -40,6 +40,12 @@ public class WebHdfsTestUtil { public static final Log LOG = LogFactory.getLog(WebHdfsTestUtil.class); + public static Configuration createConf() { + final Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); + return conf; + } + public static WebHdfsFileSystem getWebHdfsFileSystem(final Configuration conf ) throws IOException, URISyntaxException { final String uri = WebHdfsFileSystem.SCHEME + "://" From 7e01d2e06cd11bb0ea472ca192ca9496a77e4847 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Wed, 27 Jun 2012 14:28:08 +0000 Subject: [PATCH 83/91] MAPREDUCE-4372. Deadlock in Resource Manager (Devaraj K via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1354531 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../hadoop/yarn/server/resourcemanager/ResourceManager.java | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 36dc62f5f5..d80e42955a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -194,6 +194,8 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-2289. Permissions race can make getStagingDir fail on local filesystem (ahmed via tucu) + MAPREDUCE-4372. Deadlock in Resource Manager (Devaraj K via bobby) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index abf3ff9dfe..9950b933a9 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -20,7 +20,6 @@ import java.io.IOException; -import java.net.InetAddress; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -48,8 +47,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.RMState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.RMState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; @@ -327,7 +326,8 @@ public void run() { } catch (Throwable t) { LOG.fatal("Error in handling event type " + event.getType() + " to the scheduler", t); - if (shouldExitOnError) { + if (shouldExitOnError + && !ShutdownHookManager.get().isShutdownInProgress()) { LOG.info("Exiting, bbye.."); System.exit(-1); } From 54de1ddfc168e1f523d86ba398df7d1b32e8435c Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 27 Jun 2012 16:06:37 +0000 Subject: [PATCH 84/91] HDFS-3481. Refactor HttpFS handling of JAX-RS query string parameters (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1354599 13f79535-47bb-0310-9956-ffa450edef68 --- .../fs/http/client/HttpFSFileSystem.java | 149 ++-- .../server/CheckUploadContentTypeFilter.java | 7 +- .../http/server/HttpFSExceptionProvider.java | 6 + .../http/server/HttpFSParametersProvider.java | 398 ++++++++++ .../hadoop/fs/http/server/HttpFSParams.java | 551 -------------- .../hadoop/fs/http/server/HttpFSServer.java | 684 +++++++++--------- .../apache/hadoop/lib/wsrs/BooleanParam.java | 7 +- .../org/apache/hadoop/lib/wsrs/ByteParam.java | 4 +- .../org/apache/hadoop/lib/wsrs/EnumParam.java | 4 +- .../apache/hadoop/lib/wsrs/IntegerParam.java | 4 +- .../org/apache/hadoop/lib/wsrs/LongParam.java | 4 +- .../org/apache/hadoop/lib/wsrs/Param.java | 23 +- .../apache/hadoop/lib/wsrs/Parameters.java} | 48 +- .../hadoop/lib/wsrs/ParametersProvider.java | 107 +++ .../apache/hadoop/lib/wsrs/ShortParam.java | 4 +- .../apache/hadoop/lib/wsrs/StringParam.java | 22 +- .../fs/http/client/TestHttpFSFileSystem.java | 1 + .../TestCheckUploadContentTypeFilter.java | 18 +- .../hadoop/lib/wsrs/TestBooleanParam.java | 50 -- .../apache/hadoop/lib/wsrs/TestByteParam.java | 53 -- .../apache/hadoop/lib/wsrs/TestEnumParam.java | 52 -- .../hadoop/lib/wsrs/TestIntegerParam.java | 52 -- .../org/apache/hadoop/lib/wsrs/TestParam.java | 120 +++ .../hadoop/lib/wsrs/TestShortParam.java | 53 -- .../hadoop/lib/wsrs/TestStringParam.java | 64 -- .../test/resources/httpfs-log4j.properties | 22 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + 27 files changed, 1138 insertions(+), 1371 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParams.java rename hadoop-hdfs-project/hadoop-hdfs-httpfs/src/{test/java/org/apache/hadoop/lib/wsrs/TestLongParam.java => main/java/org/apache/hadoop/lib/wsrs/Parameters.java} (52%) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ParametersProvider.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestBooleanParam.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestByteParam.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestEnumParam.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestIntegerParam.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestParam.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestShortParam.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestStringParam.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/resources/httpfs-log4j.properties diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index 4a1a205551..fa28ba31c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -154,41 +154,33 @@ public static FILE_TYPE getType(FileStatus fileStatus) { public static final int HTTP_TEMPORARY_REDIRECT = 307; + private static final String HTTP_GET = "GET"; + private static final String HTTP_PUT = "PUT"; + private static final String HTTP_POST = "POST"; + private static final String HTTP_DELETE = "DELETE"; - /** - * Get operations. - */ - public enum GetOpValues { - OPEN, GETFILESTATUS, LISTSTATUS, GETHOMEDIRECTORY, GETCONTENTSUMMARY, GETFILECHECKSUM, - GETDELEGATIONTOKEN, GETFILEBLOCKLOCATIONS, INSTRUMENTATION - } + public enum Operation { + OPEN(HTTP_GET), GETFILESTATUS(HTTP_GET), LISTSTATUS(HTTP_GET), + GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET), + GETFILECHECKSUM(HTTP_GET), GETFILEBLOCKLOCATIONS(HTTP_GET), + INSTRUMENTATION(HTTP_GET), + APPEND(HTTP_POST), + CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT), + SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT), + DELETE(HTTP_DELETE); - /** - * Post operations. - */ - public static enum PostOpValues { - APPEND - } + private String httpMethod; - /** - * Put operations. - */ - public static enum PutOpValues { - CREATE, MKDIRS, RENAME, SETOWNER, SETPERMISSION, SETREPLICATION, SETTIMES, - RENEWDELEGATIONTOKEN, CANCELDELEGATIONTOKEN - } + Operation(String httpMethod) { + this.httpMethod = httpMethod; + } + + public String getMethod() { + return httpMethod; + } - /** - * Delete operations. - */ - public static enum DeleteOpValues { - DELETE } - private static final String HTTP_GET = "GET"; - private static final String HTTP_PUT = "PUT"; - private static final String HTTP_POST = "POST"; - private static final String HTTP_DELETE = "DELETE"; private AuthenticatedURL.Token authToken = new AuthenticatedURL.Token(); private URI uri; @@ -402,10 +394,12 @@ public boolean seekToNewSource(long targetPos) throws IOException { @Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { Map params = new HashMap(); - params.put(OP_PARAM, GetOpValues.OPEN.toString()); - HttpURLConnection conn = getConnection(HTTP_GET, params, f, true); + params.put(OP_PARAM, Operation.OPEN.toString()); + HttpURLConnection conn = getConnection(Operation.OPEN.getMethod(), params, + f, true); validateResponse(conn, HttpURLConnection.HTTP_OK); - return new FSDataInputStream(new HttpFSDataInputStream(conn.getInputStream(), bufferSize)); + return new FSDataInputStream( + new HttpFSDataInputStream(conn.getInputStream(), bufferSize)); } /** @@ -508,15 +502,18 @@ private FSDataOutputStream uploadData(String method, Path f, Map * @see #setPermission(Path, FsPermission) */ @Override - public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, - short replication, long blockSize, Progressable progress) throws IOException { + public FSDataOutputStream create(Path f, FsPermission permission, + boolean overwrite, int bufferSize, + short replication, long blockSize, + Progressable progress) throws IOException { Map params = new HashMap(); - params.put(OP_PARAM, PutOpValues.CREATE.toString()); + params.put(OP_PARAM, Operation.CREATE.toString()); params.put(OVERWRITE_PARAM, Boolean.toString(overwrite)); params.put(REPLICATION_PARAM, Short.toString(replication)); params.put(BLOCKSIZE_PARAM, Long.toString(blockSize)); params.put(PERMISSION_PARAM, permissionToString(permission)); - return uploadData(HTTP_PUT, f, params, bufferSize, HttpURLConnection.HTTP_CREATED); + return uploadData(Operation.CREATE.getMethod(), f, params, bufferSize, + HttpURLConnection.HTTP_CREATED); } @@ -532,10 +529,12 @@ public FSDataOutputStream create(Path f, FsPermission permission, boolean overwr * @throws IOException */ @Override - public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException { + public FSDataOutputStream append(Path f, int bufferSize, + Progressable progress) throws IOException { Map params = new HashMap(); - params.put(OP_PARAM, PostOpValues.APPEND.toString()); - return uploadData(HTTP_POST, f, params, bufferSize, HttpURLConnection.HTTP_OK); + params.put(OP_PARAM, Operation.APPEND.toString()); + return uploadData(Operation.APPEND.getMethod(), f, params, bufferSize, + HttpURLConnection.HTTP_OK); } /** @@ -545,9 +544,10 @@ public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) @Override public boolean rename(Path src, Path dst) throws IOException { Map params = new HashMap(); - params.put(OP_PARAM, PutOpValues.RENAME.toString()); + params.put(OP_PARAM, Operation.RENAME.toString()); params.put(DESTINATION_PARAM, dst.toString()); - HttpURLConnection conn = getConnection(HTTP_PUT, params, src, true); + HttpURLConnection conn = getConnection(Operation.RENAME.getMethod(), + params, src, true); validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) jsonParse(conn); return (Boolean) json.get(RENAME_JSON); @@ -580,9 +580,10 @@ public boolean delete(Path f) throws IOException { @Override public boolean delete(Path f, boolean recursive) throws IOException { Map params = new HashMap(); - params.put(OP_PARAM, DeleteOpValues.DELETE.toString()); + params.put(OP_PARAM, Operation.DELETE.toString()); params.put(RECURSIVE_PARAM, Boolean.toString(recursive)); - HttpURLConnection conn = getConnection(HTTP_DELETE, params, f, true); + HttpURLConnection conn = getConnection(Operation.DELETE.getMethod(), + params, f, true); validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) jsonParse(conn); return (Boolean) json.get(DELETE_JSON); @@ -601,8 +602,9 @@ public boolean delete(Path f, boolean recursive) throws IOException { @Override public FileStatus[] listStatus(Path f) throws IOException { Map params = new HashMap(); - params.put(OP_PARAM, GetOpValues.LISTSTATUS.toString()); - HttpURLConnection conn = getConnection(HTTP_GET, params, f, true); + params.put(OP_PARAM, Operation.LISTSTATUS.toString()); + HttpURLConnection conn = getConnection(Operation.LISTSTATUS.getMethod(), + params, f, true); validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) jsonParse(conn); json = (JSONObject) json.get(FILE_STATUSES_JSON); @@ -647,9 +649,10 @@ public Path getWorkingDirectory() { @Override public boolean mkdirs(Path f, FsPermission permission) throws IOException { Map params = new HashMap(); - params.put(OP_PARAM, PutOpValues.MKDIRS.toString()); + params.put(OP_PARAM, Operation.MKDIRS.toString()); params.put(PERMISSION_PARAM, permissionToString(permission)); - HttpURLConnection conn = getConnection(HTTP_PUT, params, f, true); + HttpURLConnection conn = getConnection(Operation.MKDIRS.getMethod(), + params, f, true); validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) jsonParse(conn); return (Boolean) json.get(MKDIRS_JSON); @@ -668,8 +671,9 @@ public boolean mkdirs(Path f, FsPermission permission) throws IOException { @Override public FileStatus getFileStatus(Path f) throws IOException { Map params = new HashMap(); - params.put(OP_PARAM, GetOpValues.GETFILESTATUS.toString()); - HttpURLConnection conn = getConnection(HTTP_GET, params, f, true); + params.put(OP_PARAM, Operation.GETFILESTATUS.toString()); + HttpURLConnection conn = getConnection(Operation.GETFILESTATUS.getMethod(), + params, f, true); validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) jsonParse(conn); json = (JSONObject) json.get(FILE_STATUS_JSON); @@ -684,9 +688,11 @@ public FileStatus getFileStatus(Path f) throws IOException { @Override public Path getHomeDirectory() { Map params = new HashMap(); - params.put(OP_PARAM, GetOpValues.GETHOMEDIRECTORY.toString()); + params.put(OP_PARAM, Operation.GETHOMEDIRECTORY.toString()); try { - HttpURLConnection conn = getConnection(HTTP_GET, params, new Path(getUri().toString(), "/"), false); + HttpURLConnection conn = + getConnection(Operation.GETHOMEDIRECTORY.getMethod(), params, + new Path(getUri().toString(), "/"), false); validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) jsonParse(conn); return new Path((String) json.get(HOME_DIR_JSON)); @@ -704,12 +710,14 @@ public Path getHomeDirectory() { * @param groupname If it is null, the original groupname remains unchanged. */ @Override - public void setOwner(Path p, String username, String groupname) throws IOException { + public void setOwner(Path p, String username, String groupname) + throws IOException { Map params = new HashMap(); - params.put(OP_PARAM, PutOpValues.SETOWNER.toString()); + params.put(OP_PARAM, Operation.SETOWNER.toString()); params.put(OWNER_PARAM, username); params.put(GROUP_PARAM, groupname); - HttpURLConnection conn = getConnection(HTTP_PUT, params, p, true); + HttpURLConnection conn = getConnection(Operation.SETOWNER.getMethod(), + params, p, true); validateResponse(conn, HttpURLConnection.HTTP_OK); } @@ -722,9 +730,9 @@ public void setOwner(Path p, String username, String groupname) throws IOExcepti @Override public void setPermission(Path p, FsPermission permission) throws IOException { Map params = new HashMap(); - params.put(OP_PARAM, PutOpValues.SETPERMISSION.toString()); + params.put(OP_PARAM, Operation.SETPERMISSION.toString()); params.put(PERMISSION_PARAM, permissionToString(permission)); - HttpURLConnection conn = getConnection(HTTP_PUT, params, p, true); + HttpURLConnection conn = getConnection(Operation.SETPERMISSION.getMethod(), params, p, true); validateResponse(conn, HttpURLConnection.HTTP_OK); } @@ -742,10 +750,11 @@ public void setPermission(Path p, FsPermission permission) throws IOException { @Override public void setTimes(Path p, long mtime, long atime) throws IOException { Map params = new HashMap(); - params.put(OP_PARAM, PutOpValues.SETTIMES.toString()); + params.put(OP_PARAM, Operation.SETTIMES.toString()); params.put(MODIFICATION_TIME_PARAM, Long.toString(mtime)); params.put(ACCESS_TIME_PARAM, Long.toString(atime)); - HttpURLConnection conn = getConnection(HTTP_PUT, params, p, true); + HttpURLConnection conn = getConnection(Operation.SETTIMES.getMethod(), + params, p, true); validateResponse(conn, HttpURLConnection.HTTP_OK); } @@ -761,11 +770,13 @@ public void setTimes(Path p, long mtime, long atime) throws IOException { * @throws IOException */ @Override - public boolean setReplication(Path src, short replication) throws IOException { + public boolean setReplication(Path src, short replication) + throws IOException { Map params = new HashMap(); - params.put(OP_PARAM, PutOpValues.SETREPLICATION.toString()); + params.put(OP_PARAM, Operation.SETREPLICATION.toString()); params.put(REPLICATION_PARAM, Short.toString(replication)); - HttpURLConnection conn = getConnection(HTTP_PUT, params, src, true); + HttpURLConnection conn = + getConnection(Operation.SETREPLICATION.getMethod(), params, src, true); validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) jsonParse(conn); return (Boolean) json.get(SET_REPLICATION_JSON); @@ -814,10 +825,12 @@ private FileStatus createFileStatus(Path parent, JSONObject json) { @Override public ContentSummary getContentSummary(Path f) throws IOException { Map params = new HashMap(); - params.put(OP_PARAM, GetOpValues.GETCONTENTSUMMARY.toString()); - HttpURLConnection conn = getConnection(HTTP_GET, params, f, true); + params.put(OP_PARAM, Operation.GETCONTENTSUMMARY.toString()); + HttpURLConnection conn = + getConnection(Operation.GETCONTENTSUMMARY.getMethod(), params, f, true); validateResponse(conn, HttpURLConnection.HTTP_OK); - JSONObject json = (JSONObject) ((JSONObject) jsonParse(conn)).get(CONTENT_SUMMARY_JSON); + JSONObject json = + (JSONObject) ((JSONObject) jsonParse(conn)).get(CONTENT_SUMMARY_JSON); return new ContentSummary((Long) json.get(CONTENT_SUMMARY_LENGTH_JSON), (Long) json.get(CONTENT_SUMMARY_FILE_COUNT_JSON), (Long) json.get(CONTENT_SUMMARY_DIRECTORY_COUNT_JSON), @@ -830,10 +843,12 @@ public ContentSummary getContentSummary(Path f) throws IOException { @Override public FileChecksum getFileChecksum(Path f) throws IOException { Map params = new HashMap(); - params.put(OP_PARAM, GetOpValues.GETFILECHECKSUM.toString()); - HttpURLConnection conn = getConnection(HTTP_GET, params, f, true); + params.put(OP_PARAM, Operation.GETFILECHECKSUM.toString()); + HttpURLConnection conn = + getConnection(Operation.GETFILECHECKSUM.getMethod(), params, f, true); validateResponse(conn, HttpURLConnection.HTTP_OK); - final JSONObject json = (JSONObject) ((JSONObject) jsonParse(conn)).get(FILE_CHECKSUM_JSON); + final JSONObject json = + (JSONObject) ((JSONObject) jsonParse(conn)).get(FILE_CHECKSUM_JSON); return new FileChecksum() { @Override public String getAlgorithmName() { diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/CheckUploadContentTypeFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/CheckUploadContentTypeFilter.java index 7e73666f58..abd382d871 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/CheckUploadContentTypeFilter.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/CheckUploadContentTypeFilter.java @@ -30,7 +30,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; -import java.net.InetAddress; import java.util.HashSet; import java.util.Set; @@ -43,8 +42,8 @@ public class CheckUploadContentTypeFilter implements Filter { private static final Set UPLOAD_OPERATIONS = new HashSet(); static { - UPLOAD_OPERATIONS.add(HttpFSFileSystem.PostOpValues.APPEND.toString()); - UPLOAD_OPERATIONS.add(HttpFSFileSystem.PutOpValues.CREATE.toString()); + UPLOAD_OPERATIONS.add(HttpFSFileSystem.Operation.APPEND.toString()); + UPLOAD_OPERATIONS.add(HttpFSFileSystem.Operation.CREATE.toString()); } /** @@ -82,7 +81,7 @@ public void doFilter(ServletRequest request, ServletResponse response, if (method.equals("PUT") || method.equals("POST")) { String op = httpReq.getParameter(HttpFSFileSystem.OP_PARAM); if (op != null && UPLOAD_OPERATIONS.contains(op.toUpperCase())) { - if ("true".equalsIgnoreCase(httpReq.getParameter(HttpFSParams.DataParam.NAME))) { + if ("true".equalsIgnoreCase(httpReq.getParameter(HttpFSParametersProvider.DataParam.NAME))) { String contentType = httpReq.getContentType(); contentTypeOK = HttpFSFileSystem.UPLOAD_CONTENT_TYPE.equalsIgnoreCase(contentType); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java index 26dff496dd..b999a72557 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.http.server; +import com.sun.jersey.api.container.ContainerException; import org.apache.hadoop.lib.service.FileSystemAccessException; import org.apache.hadoop.lib.wsrs.ExceptionProvider; import org.slf4j.Logger; @@ -59,6 +60,9 @@ public Response toResponse(Throwable throwable) { if (throwable instanceof FileSystemAccessException) { throwable = throwable.getCause(); } + if (throwable instanceof ContainerException) { + throwable = throwable.getCause(); + } if (throwable instanceof SecurityException) { status = Response.Status.UNAUTHORIZED; } else if (throwable instanceof FileNotFoundException) { @@ -67,6 +71,8 @@ public Response toResponse(Throwable throwable) { status = Response.Status.INTERNAL_SERVER_ERROR; } else if (throwable instanceof UnsupportedOperationException) { status = Response.Status.BAD_REQUEST; + } else if (throwable instanceof IllegalArgumentException) { + status = Response.Status.BAD_REQUEST; } else { status = Response.Status.INTERNAL_SERVER_ERROR; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java new file mode 100644 index 0000000000..0ab10179c8 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java @@ -0,0 +1,398 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.http.server; + +import org.apache.hadoop.fs.http.client.HttpFSFileSystem; +import org.apache.hadoop.fs.http.client.HttpFSFileSystem.Operation; +import org.apache.hadoop.lib.wsrs.BooleanParam; +import org.apache.hadoop.lib.wsrs.EnumParam; +import org.apache.hadoop.lib.wsrs.LongParam; +import org.apache.hadoop.lib.wsrs.Param; +import org.apache.hadoop.lib.wsrs.ParametersProvider; +import org.apache.hadoop.lib.wsrs.ShortParam; +import org.apache.hadoop.lib.wsrs.StringParam; +import org.apache.hadoop.lib.wsrs.UserProvider; +import org.slf4j.MDC; + +import javax.ws.rs.ext.Provider; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * HttpFS ParametersProvider. + */ +@Provider +public class HttpFSParametersProvider extends ParametersProvider { + + private static final Map>[]> PARAMS_DEF = + new HashMap>[]>(); + + static { + PARAMS_DEF.put(Operation.OPEN, + new Class[]{DoAsParam.class, OffsetParam.class, LenParam.class}); + PARAMS_DEF.put(Operation.GETFILESTATUS, new Class[]{DoAsParam.class}); + PARAMS_DEF.put(Operation.LISTSTATUS, + new Class[]{DoAsParam.class, FilterParam.class}); + PARAMS_DEF.put(Operation.GETHOMEDIRECTORY, new Class[]{DoAsParam.class}); + PARAMS_DEF.put(Operation.GETCONTENTSUMMARY, new Class[]{DoAsParam.class}); + PARAMS_DEF.put(Operation.GETFILECHECKSUM, new Class[]{DoAsParam.class}); + PARAMS_DEF.put(Operation.GETFILEBLOCKLOCATIONS, + new Class[]{DoAsParam.class}); + PARAMS_DEF.put(Operation.INSTRUMENTATION, new Class[]{DoAsParam.class}); + PARAMS_DEF.put(Operation.APPEND, + new Class[]{DoAsParam.class, DataParam.class}); + PARAMS_DEF.put(Operation.CREATE, + new Class[]{DoAsParam.class, PermissionParam.class, OverwriteParam.class, + ReplicationParam.class, BlockSizeParam.class, DataParam.class}); + PARAMS_DEF.put(Operation.MKDIRS, + new Class[]{DoAsParam.class, PermissionParam.class}); + PARAMS_DEF.put(Operation.RENAME, + new Class[]{DoAsParam.class, DestinationParam.class}); + PARAMS_DEF.put(Operation.SETOWNER, + new Class[]{DoAsParam.class, OwnerParam.class, GroupParam.class}); + PARAMS_DEF.put(Operation.SETPERMISSION, + new Class[]{DoAsParam.class, PermissionParam.class}); + PARAMS_DEF.put(Operation.SETREPLICATION, + new Class[]{DoAsParam.class, ReplicationParam.class}); + PARAMS_DEF.put(Operation.SETTIMES, + new Class[]{DoAsParam.class, ModifiedTimeParam.class, + AccessTimeParam.class}); + PARAMS_DEF.put(Operation.DELETE, + new Class[]{DoAsParam.class, RecursiveParam.class}); + } + + public HttpFSParametersProvider() { + super(HttpFSFileSystem.OP_PARAM, HttpFSFileSystem.Operation.class, + PARAMS_DEF); + } + + /** + * Class for access-time parameter. + */ + public static class AccessTimeParam extends LongParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.ACCESS_TIME_PARAM; + /** + * Constructor. + */ + public AccessTimeParam() { + super(NAME, -1l); + } + } + + /** + * Class for block-size parameter. + */ + public static class BlockSizeParam extends LongParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.BLOCKSIZE_PARAM; + + /** + * Constructor. + */ + public BlockSizeParam() { + super(NAME, -1l); + } + } + + /** + * Class for data parameter. + */ + public static class DataParam extends BooleanParam { + + /** + * Parameter name. + */ + public static final String NAME = "data"; + + /** + * Constructor. + */ + public DataParam() { + super(NAME, false); + } + } + + /** + * Class for operation parameter. + */ + public static class OperationParam extends EnumParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.OP_PARAM; + /** + * Constructor. + */ + public OperationParam(String operation) { + super(NAME, HttpFSFileSystem.Operation.class, + HttpFSFileSystem.Operation.valueOf(operation.toUpperCase())); + } + } + + /** + * Class for delete's recursive parameter. + */ + public static class RecursiveParam extends BooleanParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.RECURSIVE_PARAM; + + /** + * Constructor. + */ + public RecursiveParam() { + super(NAME, false); + } + } + + /** + * Class for do-as parameter. + */ + public static class DoAsParam extends StringParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.DO_AS_PARAM; + + /** + * Constructor. + */ + public DoAsParam() { + super(NAME, null, UserProvider.USER_PATTERN); + } + + /** + * Delegates to parent and then adds do-as user to + * MDC context for logging purposes. + * + * + * @param str parameter value. + * + * @return parsed parameter + */ + @Override + public String parseParam(String str) { + String doAs = super.parseParam(str); + MDC.put(getName(), (doAs != null) ? doAs : "-"); + return doAs; + } + } + + /** + * Class for filter parameter. + */ + public static class FilterParam extends StringParam { + + /** + * Parameter name. + */ + public static final String NAME = "filter"; + + /** + * Constructor. + */ + public FilterParam() { + super(NAME, null); + } + + } + + /** + * Class for group parameter. + */ + public static class GroupParam extends StringParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.GROUP_PARAM; + + /** + * Constructor. + */ + public GroupParam() { + super(NAME, null, UserProvider.USER_PATTERN); + } + + } + + /** + * Class for len parameter. + */ + public static class LenParam extends LongParam { + + /** + * Parameter name. + */ + public static final String NAME = "len"; + + /** + * Constructor. + */ + public LenParam() { + super(NAME, -1l); + } + } + + /** + * Class for modified-time parameter. + */ + public static class ModifiedTimeParam extends LongParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.MODIFICATION_TIME_PARAM; + + /** + * Constructor. + */ + public ModifiedTimeParam() { + super(NAME, -1l); + } + } + + /** + * Class for offset parameter. + */ + public static class OffsetParam extends LongParam { + + /** + * Parameter name. + */ + public static final String NAME = "offset"; + + /** + * Constructor. + */ + public OffsetParam() { + super(NAME, 0l); + } + } + + /** + * Class for overwrite parameter. + */ + public static class OverwriteParam extends BooleanParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.OVERWRITE_PARAM; + + /** + * Constructor. + */ + public OverwriteParam() { + super(NAME, true); + } + } + + /** + * Class for owner parameter. + */ + public static class OwnerParam extends StringParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.OWNER_PARAM; + + /** + * Constructor. + */ + public OwnerParam() { + super(NAME, null, UserProvider.USER_PATTERN); + } + + } + + /** + * Class for permission parameter. + */ + public static class PermissionParam extends StringParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.PERMISSION_PARAM; + + /** + * Symbolic Unix permissions regular expression pattern. + */ + private static final Pattern PERMISSION_PATTERN = + Pattern.compile(HttpFSFileSystem.DEFAULT_PERMISSION + + "|[0-1]?[0-7][0-7][0-7]"); + + /** + * Constructor. + */ + public PermissionParam() { + super(NAME, HttpFSFileSystem.DEFAULT_PERMISSION, PERMISSION_PATTERN); + } + + } + + /** + * Class for replication parameter. + */ + public static class ReplicationParam extends ShortParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.REPLICATION_PARAM; + + /** + * Constructor. + */ + public ReplicationParam() { + super(NAME, (short) -1); + } + } + + /** + * Class for to-path parameter. + */ + public static class DestinationParam extends StringParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.DESTINATION_PARAM; + + /** + * Constructor. + */ + public DestinationParam() { + super(NAME, null); + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParams.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParams.java deleted file mode 100644 index 3c7b5f7499..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParams.java +++ /dev/null @@ -1,551 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.fs.http.server; - -import org.apache.hadoop.fs.http.client.HttpFSFileSystem; -import org.apache.hadoop.lib.wsrs.BooleanParam; -import org.apache.hadoop.lib.wsrs.EnumParam; -import org.apache.hadoop.lib.wsrs.LongParam; -import org.apache.hadoop.lib.wsrs.ShortParam; -import org.apache.hadoop.lib.wsrs.StringParam; -import org.apache.hadoop.lib.wsrs.UserProvider; -import org.slf4j.MDC; - -import java.util.regex.Pattern; - -/** - * HttpFS HTTP Parameters used by {@link HttpFSServer}. - */ -public class HttpFSParams { - - /** - * To avoid instantiation. - */ - private HttpFSParams() { - } - - /** - * Class for access-time parameter. - */ - public static class AccessTimeParam extends LongParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.ACCESS_TIME_PARAM; - - /** - * Default parameter value. - */ - public static final String DEFAULT = "-1"; - - /** - * Constructor. - * - * @param str parameter value. - */ - public AccessTimeParam(String str) { - super(NAME, str); - } - } - - /** - * Class for block-size parameter. - */ - public static class BlockSizeParam extends LongParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.BLOCKSIZE_PARAM; - - /** - * Default parameter value. - */ - public static final String DEFAULT = "-1"; - - /** - * Constructor. - * - * @param str parameter value. - */ - public BlockSizeParam(String str) { - super(NAME, str); - } - } - - /** - * Class for data parameter. - */ - public static class DataParam extends BooleanParam { - - /** - * Parameter name. - */ - public static final String NAME = "data"; - - /** - * Default parameter value. - */ - public static final String DEFAULT = "false"; - - /** - * Constructor. - * - * @param str parameter value. - */ - public DataParam(String str) { - super(NAME, str); - } - } - - /** - * Class for DELETE operation parameter. - */ - public static class DeleteOpParam extends EnumParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.OP_PARAM; - - /** - * Constructor. - * - * @param str parameter value. - */ - public DeleteOpParam(String str) { - super(NAME, str, HttpFSFileSystem.DeleteOpValues.class); - } - } - - /** - * Class for delete's recursive parameter. - */ - public static class DeleteRecursiveParam extends BooleanParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.RECURSIVE_PARAM; - - /** - * Default parameter value. - */ - public static final String DEFAULT = "false"; - - /** - * Constructor. - * - * @param str parameter value. - */ - public DeleteRecursiveParam(String str) { - super(NAME, str); - } - } - - /** - * Class for do-as parameter. - */ - public static class DoAsParam extends StringParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.DO_AS_PARAM; - - /** - * Default parameter value. - */ - public static final String DEFAULT = ""; - - /** - * Constructor. - * - * @param str parameter value. - */ - public DoAsParam(String str) { - super(NAME, str, UserProvider.USER_PATTERN); - } - - /** - * Delegates to parent and then adds do-as user to - * MDC context for logging purposes. - * - * @param name parameter name. - * @param str parameter value. - * - * @return parsed parameter - */ - @Override - public String parseParam(String name, String str) { - String doAs = super.parseParam(name, str); - MDC.put(NAME, (doAs != null) ? doAs : "-"); - return doAs; - } - } - - /** - * Class for filter parameter. - */ - public static class FilterParam extends StringParam { - - /** - * Parameter name. - */ - public static final String NAME = "filter"; - - /** - * Default parameter value. - */ - public static final String DEFAULT = ""; - - /** - * Constructor. - * - * @param expr parameter value. - */ - public FilterParam(String expr) { - super(NAME, expr); - } - - } - - /** - * Class for path parameter. - */ - public static class FsPathParam extends StringParam { - - /** - * Constructor. - * - * @param path parameter value. - */ - public FsPathParam(String path) { - super("path", path); - } - - /** - * Makes the path absolute adding '/' to it. - *

- * This is required because JAX-RS resolution of paths does not add - * the root '/'. - */ - public void makeAbsolute() { - String path = value(); - path = "/" + ((path != null) ? path : ""); - setValue(path); - } - - } - - /** - * Class for GET operation parameter. - */ - public static class GetOpParam extends EnumParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.OP_PARAM; - - /** - * Constructor. - * - * @param str parameter value. - */ - public GetOpParam(String str) { - super(NAME, str, HttpFSFileSystem.GetOpValues.class); - } - } - - /** - * Class for group parameter. - */ - public static class GroupParam extends StringParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.GROUP_PARAM; - - /** - * Default parameter value. - */ - public static final String DEFAULT = ""; - - /** - * Constructor. - * - * @param str parameter value. - */ - public GroupParam(String str) { - super(NAME, str, UserProvider.USER_PATTERN); - } - - } - - /** - * Class for len parameter. - */ - public static class LenParam extends LongParam { - - /** - * Parameter name. - */ - public static final String NAME = "len"; - - /** - * Default parameter value. - */ - public static final String DEFAULT = "-1"; - - /** - * Constructor. - * - * @param str parameter value. - */ - public LenParam(String str) { - super(NAME, str); - } - } - - /** - * Class for modified-time parameter. - */ - public static class ModifiedTimeParam extends LongParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.MODIFICATION_TIME_PARAM; - - /** - * Default parameter value. - */ - public static final String DEFAULT = "-1"; - - /** - * Constructor. - * - * @param str parameter value. - */ - public ModifiedTimeParam(String str) { - super(NAME, str); - } - } - - /** - * Class for offset parameter. - */ - public static class OffsetParam extends LongParam { - - /** - * Parameter name. - */ - public static final String NAME = "offset"; - - /** - * Default parameter value. - */ - public static final String DEFAULT = "0"; - - /** - * Constructor. - * - * @param str parameter value. - */ - public OffsetParam(String str) { - super(NAME, str); - } - } - - /** - * Class for overwrite parameter. - */ - public static class OverwriteParam extends BooleanParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.OVERWRITE_PARAM; - - /** - * Default parameter value. - */ - public static final String DEFAULT = "true"; - - /** - * Constructor. - * - * @param str parameter value. - */ - public OverwriteParam(String str) { - super(NAME, str); - } - } - - /** - * Class for owner parameter. - */ - public static class OwnerParam extends StringParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.OWNER_PARAM; - - /** - * Default parameter value. - */ - public static final String DEFAULT = ""; - - /** - * Constructor. - * - * @param str parameter value. - */ - public OwnerParam(String str) { - super(NAME, str, UserProvider.USER_PATTERN); - } - - } - - /** - * Class for permission parameter. - */ - public static class PermissionParam extends StringParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.PERMISSION_PARAM; - - /** - * Default parameter value. - */ - public static final String DEFAULT = HttpFSFileSystem.DEFAULT_PERMISSION; - - - /** - * Symbolic Unix permissions regular expression pattern. - */ - private static final Pattern PERMISSION_PATTERN = - Pattern.compile(DEFAULT + "|[0-1]?[0-7][0-7][0-7]"); - - /** - * Constructor. - * - * @param permission parameter value. - */ - public PermissionParam(String permission) { - super(NAME, permission.toLowerCase(), PERMISSION_PATTERN); - } - - } - - /** - * Class for POST operation parameter. - */ - public static class PostOpParam extends EnumParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.OP_PARAM; - - /** - * Constructor. - * - * @param str parameter value. - */ - public PostOpParam(String str) { - super(NAME, str, HttpFSFileSystem.PostOpValues.class); - } - } - - /** - * Class for PUT operation parameter. - */ - public static class PutOpParam extends EnumParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.OP_PARAM; - - /** - * Constructor. - * - * @param str parameter value. - */ - public PutOpParam(String str) { - super(NAME, str, HttpFSFileSystem.PutOpValues.class); - } - } - - /** - * Class for replication parameter. - */ - public static class ReplicationParam extends ShortParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.REPLICATION_PARAM; - - /** - * Default parameter value. - */ - public static final String DEFAULT = "-1"; - - /** - * Constructor. - * - * @param str parameter value. - */ - public ReplicationParam(String str) { - super(NAME, str); - } - } - - /** - * Class for to-path parameter. - */ - public static class ToPathParam extends StringParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.DESTINATION_PARAM; - - /** - * Default parameter value. - */ - public static final String DEFAULT = ""; - - /** - * Constructor. - * - * @param path parameter value. - */ - public ToPathParam(String path) { - super(NAME, path); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java index cf9048528b..22a173ac8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java @@ -21,26 +21,22 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.http.client.HttpFSFileSystem; -import org.apache.hadoop.fs.http.server.HttpFSParams.AccessTimeParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.BlockSizeParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.DataParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.DeleteOpParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.DeleteRecursiveParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.DoAsParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.FilterParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.FsPathParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.GetOpParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.GroupParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.LenParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.ModifiedTimeParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.OffsetParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.OverwriteParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.OwnerParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.PermissionParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.PostOpParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.PutOpParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.ReplicationParam; -import org.apache.hadoop.fs.http.server.HttpFSParams.ToPathParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OperationParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.AccessTimeParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.BlockSizeParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DataParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.RecursiveParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DoAsParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.FilterParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.GroupParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.LenParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.ModifiedTimeParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OffsetParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OverwriteParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OwnerParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.PermissionParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.ReplicationParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DestinationParam; import org.apache.hadoop.lib.service.FileSystemAccess; import org.apache.hadoop.lib.service.FileSystemAccessException; import org.apache.hadoop.lib.service.Groups; @@ -49,6 +45,7 @@ import org.apache.hadoop.lib.servlet.FileSystemReleaseFilter; import org.apache.hadoop.lib.servlet.HostnameFilter; import org.apache.hadoop.lib.wsrs.InputStreamEntity; +import org.apache.hadoop.lib.wsrs.Parameters; import org.apache.hadoop.security.authentication.server.AuthenticationToken; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -57,7 +54,6 @@ import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; -import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.PUT; @@ -89,39 +85,6 @@ public class HttpFSServer { private static Logger AUDIT_LOG = LoggerFactory.getLogger("httpfsaudit"); - /** - * Special binding for '/' as it is not handled by the wildcard binding. - * - * @param user principal making the request. - * @param op GET operation, default value is @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.GetOpValues#OPEN}. - * @param filter Glob filter, default value is none. Used only if the - * operation is @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.GetOpValues#LISTSTATUS} - * @param doAs user being impersonated, defualt value is none. It can be used - * only if the current user is a HttpFSServer proxyuser. - * - * @return the request response - * - * @throws IOException thrown if an IO error occurred. Thrown exceptions are - * handled by {@link HttpFSExceptionProvider}. - * @throws FileSystemAccessException thrown if a FileSystemAccess releated error occurred. Thrown - * exceptions are handled by {@link HttpFSExceptionProvider}. - */ - @GET - @Path("/") - @Produces(MediaType.APPLICATION_JSON) - public Response root(@Context Principal user, - @QueryParam(GetOpParam.NAME) GetOpParam op, - @QueryParam(FilterParam.NAME) @DefaultValue(FilterParam.DEFAULT) FilterParam filter, - @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) DoAsParam doAs) - throws IOException, FileSystemAccessException { - return get(user, new FsPathParam(""), op, new OffsetParam(OffsetParam.DEFAULT), - new LenParam(LenParam.DEFAULT), filter, doAs, - new OverwriteParam(OverwriteParam.DEFAULT), - new BlockSizeParam(BlockSizeParam.DEFAULT), - new PermissionParam(PermissionParam.DEFAULT), - new ReplicationParam(ReplicationParam.DEFAULT)); - } - /** * Resolves the effective user that will be used to request a FileSystemAccess filesystem. *

@@ -207,402 +170,405 @@ private FileSystem createFileSystem(Principal user, String doAs) throws IOExcept return fs; } + private void enforceRootPath(HttpFSFileSystem.Operation op, String path) { + if (!path.equals("/")) { + throw new UnsupportedOperationException( + MessageFormat.format("Operation [{0}], invalid path [{1}], must be '/'", + op, path)); + } + } + /** - * Binding to handle all GET requests, supported operations are - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.GetOpValues}. - *

- * The @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.GetOpValues#INSTRUMENTATION} operation is available only - * to users that are in HttpFSServer's admin group (see {@link HttpFSServer}. It returns - * HttpFSServer instrumentation data. The specified path must be '/'. + * Special binding for '/' as it is not handled by the wildcard binding. * - * @param user principal making the request. - * @param path path for the GET request. - * @param op GET operation, default value is @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.GetOpValues#OPEN}. - * @param offset of the file being fetch, used only with - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.GetOpValues#OPEN} operations. - * @param len amounts of bytes, used only with @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.GetOpValues#OPEN} - * operations. - * @param filter Glob filter, default value is none. Used only if the - * operation is @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.GetOpValues#LISTSTATUS} - * @param doAs user being impersonated, defualt value is none. It can be used - * only if the current user is a HttpFSServer proxyuser. - * @param override default is true. Used only for - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues#CREATE} operations. - * @param blockSize block size to set, used only by - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues#CREATE} operations. - * @param permission permission to set, used only by - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues#SETPERMISSION}. - * @param replication replication factor to set, used only by - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues#SETREPLICATION}. + * @param user the principal of the user making the request. + * @param op the HttpFS operation of the request. + * @param params the HttpFS parameters of the request. * * @return the request response. * * @throws IOException thrown if an IO error occurred. Thrown exceptions are * handled by {@link HttpFSExceptionProvider}. - * @throws FileSystemAccessException thrown if a FileSystemAccess releated error occurred. Thrown - * exceptions are handled by {@link HttpFSExceptionProvider}. + * @throws FileSystemAccessException thrown if a FileSystemAccess releated + * error occurred. Thrown exceptions are handled by + * {@link HttpFSExceptionProvider}. + */ + @GET + @Path("/") + @Produces(MediaType.APPLICATION_JSON) + public Response getRoot(@Context Principal user, + @QueryParam(OperationParam.NAME) OperationParam op, + @Context Parameters params) + throws IOException, FileSystemAccessException { + return get(user, "", op, params); + } + + private String makeAbsolute(String path) { + return "/" + ((path != null) ? path : ""); + } + + /** + * Binding to handle GET requests, supported operations are + * + * @param user the principal of the user making the request. + * @param path the path for operation. + * @param op the HttpFS operation of the request. + * @param params the HttpFS parameters of the request. + * + * @return the request response. + * + * @throws IOException thrown if an IO error occurred. Thrown exceptions are + * handled by {@link HttpFSExceptionProvider}. + * @throws FileSystemAccessException thrown if a FileSystemAccess releated + * error occurred. Thrown exceptions are handled by + * {@link HttpFSExceptionProvider}. */ @GET @Path("{path:.*}") @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) public Response get(@Context Principal user, - @PathParam("path") @DefaultValue("") FsPathParam path, - @QueryParam(GetOpParam.NAME) GetOpParam op, - @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) OffsetParam offset, - @QueryParam(LenParam.NAME) @DefaultValue(LenParam.DEFAULT) LenParam len, - @QueryParam(FilterParam.NAME) @DefaultValue(FilterParam.DEFAULT) FilterParam filter, - @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) DoAsParam doAs, - - //these params are only for createHandle operation acceptance purposes - @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT) OverwriteParam override, - @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT) BlockSizeParam blockSize, - @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT) - PermissionParam permission, - @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT) - ReplicationParam replication - ) + @PathParam("path") String path, + @QueryParam(OperationParam.NAME) OperationParam op, + @Context Parameters params) throws IOException, FileSystemAccessException { - Response response = null; - if (op == null) { - throw new UnsupportedOperationException(MessageFormat.format("Missing [{0}] parameter", GetOpParam.NAME)); - } else { - path.makeAbsolute(); - MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); - switch (op.value()) { - case OPEN: { - //Invoking the command directly using an unmanaged FileSystem that is released by the - //FileSystemReleaseFilter - FSOperations.FSOpen command = new FSOperations.FSOpen(path.value()); - FileSystem fs = createFileSystem(user, doAs.value()); - InputStream is = command.execute(fs); - AUDIT_LOG.info("[{}] offset [{}] len [{}]", new Object[]{path, offset, len}); - InputStreamEntity entity = new InputStreamEntity(is, offset.value(), len.value()); - response = Response.ok(entity).type(MediaType.APPLICATION_OCTET_STREAM).build(); - break; - } - case GETFILESTATUS: { - FSOperations.FSFileStatus command = new FSOperations.FSFileStatus(path.value()); - Map json = fsExecute(user, doAs.value(), command); - AUDIT_LOG.info("[{}]", path); - response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); - break; - } - case LISTSTATUS: { - FSOperations.FSListStatus command = new FSOperations.FSListStatus(path.value(), filter.value()); - Map json = fsExecute(user, doAs.value(), command); - if (filter.value() == null) { - AUDIT_LOG.info("[{}]", path); - } else { - AUDIT_LOG.info("[{}] filter [{}]", path, filter.value()); - } - response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); - break; - } - case GETHOMEDIRECTORY: { - FSOperations.FSHomeDir command = new FSOperations.FSHomeDir(); - JSONObject json = fsExecute(user, doAs.value(), command); - AUDIT_LOG.info(""); - response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); - break; - } - case INSTRUMENTATION: { - if (!path.value().equals("/")) { - throw new UnsupportedOperationException( - MessageFormat.format("Invalid path for {0}={1}, must be '/'", - GetOpParam.NAME, HttpFSFileSystem.GetOpValues.INSTRUMENTATION)); - } - Groups groups = HttpFSServerWebApp.get().get(Groups.class); - List userGroups = groups.getGroups(user.getName()); - if (!userGroups.contains(HttpFSServerWebApp.get().getAdminGroup())) { - throw new AccessControlException("User not in HttpFSServer admin group"); - } - Instrumentation instrumentation = HttpFSServerWebApp.get().get(Instrumentation.class); - Map snapshot = instrumentation.getSnapshot(); - response = Response.ok(snapshot).build(); - break; - } - case GETCONTENTSUMMARY: { - FSOperations.FSContentSummary command = new FSOperations.FSContentSummary(path.value()); - Map json = fsExecute(user, doAs.value(), command); - AUDIT_LOG.info("[{}]", path); - response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); - break; - } - case GETFILECHECKSUM: { - FSOperations.FSFileChecksum command = new FSOperations.FSFileChecksum(path.value()); - Map json = fsExecute(user, doAs.value(), command); - AUDIT_LOG.info("[{}]", path); - response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); - break; - } - case GETDELEGATIONTOKEN: { - response = Response.status(Response.Status.BAD_REQUEST).build(); - break; - } - case GETFILEBLOCKLOCATIONS: { - response = Response.status(Response.Status.BAD_REQUEST).build(); - break; + Response response; + path = makeAbsolute(path); + MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); + String doAs = params.get(DoAsParam.NAME, DoAsParam.class); + switch (op.value()) { + case OPEN: { + //Invoking the command directly using an unmanaged FileSystem that is + // released by the FileSystemReleaseFilter + FSOperations.FSOpen command = new FSOperations.FSOpen(path); + FileSystem fs = createFileSystem(user, doAs); + InputStream is = command.execute(fs); + Long offset = params.get(OffsetParam.NAME, OffsetParam.class); + Long len = params.get(LenParam.NAME, LenParam.class); + AUDIT_LOG.info("[{}] offset [{}] len [{}]", + new Object[]{path, offset, len}); + InputStreamEntity entity = new InputStreamEntity(is, offset, len); + response = + Response.ok(entity).type(MediaType.APPLICATION_OCTET_STREAM).build(); + break; + } + case GETFILESTATUS: { + FSOperations.FSFileStatus command = + new FSOperations.FSFileStatus(path); + Map json = fsExecute(user, doAs, command); + AUDIT_LOG.info("[{}]", path); + response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); + break; + } + case LISTSTATUS: { + String filter = params.get(FilterParam.NAME, FilterParam.class); + FSOperations.FSListStatus command = new FSOperations.FSListStatus( + path, filter); + Map json = fsExecute(user, doAs, command); + AUDIT_LOG.info("[{}] filter [{}]", path, + (filter != null) ? filter : "-"); + response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); + break; + } + case GETHOMEDIRECTORY: { + enforceRootPath(op.value(), path); + FSOperations.FSHomeDir command = new FSOperations.FSHomeDir(); + JSONObject json = fsExecute(user, doAs, command); + AUDIT_LOG.info(""); + response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); + break; + } + case INSTRUMENTATION: { + enforceRootPath(op.value(), path); + Groups groups = HttpFSServerWebApp.get().get(Groups.class); + List userGroups = groups.getGroups(user.getName()); + if (!userGroups.contains(HttpFSServerWebApp.get().getAdminGroup())) { + throw new AccessControlException( + "User not in HttpFSServer admin group"); } + Instrumentation instrumentation = + HttpFSServerWebApp.get().get(Instrumentation.class); + Map snapshot = instrumentation.getSnapshot(); + response = Response.ok(snapshot).build(); + break; + } + case GETCONTENTSUMMARY: { + FSOperations.FSContentSummary command = + new FSOperations.FSContentSummary(path); + Map json = fsExecute(user, doAs, command); + AUDIT_LOG.info("[{}]", path); + response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); + break; + } + case GETFILECHECKSUM: { + FSOperations.FSFileChecksum command = + new FSOperations.FSFileChecksum(path); + Map json = fsExecute(user, doAs, command); + AUDIT_LOG.info("[{}]", path); + response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); + break; + } + case GETFILEBLOCKLOCATIONS: { + response = Response.status(Response.Status.BAD_REQUEST).build(); + break; + } + default: { + throw new IOException( + MessageFormat.format("Invalid HTTP GET operation [{0}]", + op.value())); } - return response; } + return response; } - /** - * Creates the URL for an upload operation (create or append). - * - * @param uriInfo uri info of the request. - * @param uploadOperation operation for the upload URL. - * - * @return the URI for uploading data. - */ - protected URI createUploadRedirectionURL(UriInfo uriInfo, Enum uploadOperation) { - UriBuilder uriBuilder = uriInfo.getRequestUriBuilder(); - uriBuilder = uriBuilder.replaceQueryParam(PutOpParam.NAME, uploadOperation). - queryParam(DataParam.NAME, Boolean.TRUE); - return uriBuilder.build(null); - } /** - * Binding to handle all DELETE requests. + * Binding to handle DELETE requests. * - * @param user principal making the request. - * @param path path for the DELETE request. - * @param op DELETE operation, default value is @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.DeleteOpValues#DELETE}. - * @param recursive indicates if the delete is recursive, default is false - * @param doAs user being impersonated, defualt value is none. It can be used - * only if the current user is a HttpFSServer proxyuser. + * @param user the principal of the user making the request. + * @param path the path for operation. + * @param op the HttpFS operation of the request. + * @param params the HttpFS parameters of the request. * * @return the request response. * * @throws IOException thrown if an IO error occurred. Thrown exceptions are * handled by {@link HttpFSExceptionProvider}. - * @throws FileSystemAccessException thrown if a FileSystemAccess releated error occurred. Thrown - * exceptions are handled by {@link HttpFSExceptionProvider}. + * @throws FileSystemAccessException thrown if a FileSystemAccess releated + * error occurred. Thrown exceptions are handled by + * {@link HttpFSExceptionProvider}. */ @DELETE @Path("{path:.*}") @Produces(MediaType.APPLICATION_JSON) public Response delete(@Context Principal user, - @PathParam("path") FsPathParam path, - @QueryParam(DeleteOpParam.NAME) DeleteOpParam op, - @QueryParam(DeleteRecursiveParam.NAME) @DefaultValue(DeleteRecursiveParam.DEFAULT) - DeleteRecursiveParam recursive, - @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) DoAsParam doAs) + @PathParam("path") String path, + @QueryParam(OperationParam.NAME) OperationParam op, + @Context Parameters params) throws IOException, FileSystemAccessException { - Response response = null; - if (op == null) { - throw new UnsupportedOperationException(MessageFormat.format("Missing [{0}] parameter", DeleteOpParam.NAME)); - } + Response response; + path = makeAbsolute(path); + MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); + String doAs = params.get(DoAsParam.NAME, DoAsParam.class); switch (op.value()) { case DELETE: { - path.makeAbsolute(); - MDC.put(HttpFSFileSystem.OP_PARAM, "DELETE"); + Boolean recursive = + params.get(RecursiveParam.NAME, RecursiveParam.class); AUDIT_LOG.info("[{}] recursive [{}]", path, recursive); - FSOperations.FSDelete command = new FSOperations.FSDelete(path.value(), recursive.value()); - JSONObject json = fsExecute(user, doAs.value(), command); + FSOperations.FSDelete command = + new FSOperations.FSDelete(path, recursive); + JSONObject json = fsExecute(user, doAs, command); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } + default: { + throw new IOException( + MessageFormat.format("Invalid HTTP DELETE operation [{0}]", + op.value())); + } } return response; } + /** + * Binding to handle POST requests. + * + * @param is the inputstream for the request payload. + * @param user the principal of the user making the request. + * @param uriInfo the of the request. + * @param path the path for operation. + * @param op the HttpFS operation of the request. + * @param params the HttpFS parameters of the request. + * + * @return the request response. + * + * @throws IOException thrown if an IO error occurred. Thrown exceptions are + * handled by {@link HttpFSExceptionProvider}. + * @throws FileSystemAccessException thrown if a FileSystemAccess releated + * error occurred. Thrown exceptions are handled by + * {@link HttpFSExceptionProvider}. + */ + @POST + @Path("{path:.*}") + @Consumes({"*/*"}) + @Produces({MediaType.APPLICATION_JSON}) + public Response post(InputStream is, + @Context Principal user, + @Context UriInfo uriInfo, + @PathParam("path") String path, + @QueryParam(OperationParam.NAME) OperationParam op, + @Context Parameters params) + throws IOException, FileSystemAccessException { + Response response; + path = makeAbsolute(path); + MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); + String doAs = params.get(DoAsParam.NAME, DoAsParam.class); + switch (op.value()) { + case APPEND: { + boolean hasData = params.get(DataParam.NAME, DataParam.class); + if (!hasData) { + response = Response.temporaryRedirect( + createUploadRedirectionURL(uriInfo, + HttpFSFileSystem.Operation.APPEND)).build(); + } else { + FSOperations.FSAppend command = + new FSOperations.FSAppend(is, path); + fsExecute(user, doAs, command); + AUDIT_LOG.info("[{}]", path); + response = Response.ok().type(MediaType.APPLICATION_JSON).build(); + } + break; + } + default: { + throw new IOException( + MessageFormat.format("Invalid HTTP POST operation [{0}]", + op.value())); + } + } + return response; + } /** - * Binding to handle all PUT requests, supported operations are - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues}. + * Creates the URL for an upload operation (create or append). * - * @param is request input stream, used only for - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PostOpValues#APPEND} operations. - * @param user principal making the request. - * @param uriInfo the request uriInfo. - * @param path path for the PUT request. - * @param op PUT operation, no default value. - * @param toPath new path, used only for - * {@link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues#RENAME} operations. - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues#SETTIMES}. - * @param owner owner to set, used only for - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues#SETOWNER} operations. - * @param group group to set, used only for - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues#SETOWNER} operations. - * @param override default is true. Used only for - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues#CREATE} operations. - * @param blockSize block size to set, used only by - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues#CREATE} operations. - * @param permission permission to set, used only by - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues#SETPERMISSION}. - * @param replication replication factor to set, used only by - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues#SETREPLICATION}. - * @param modifiedTime modified time, in seconds since EPOC, used only by - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues#SETTIMES}. - * @param accessTime accessed time, in seconds since EPOC, used only by - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PutOpValues#SETTIMES}. - * @param hasData indicates if the append request is uploading data or not - * (just getting the handle). - * @param doAs user being impersonated, defualt value is none. It can be used - * only if the current user is a HttpFSServer proxyuser. + * @param uriInfo uri info of the request. + * @param uploadOperation operation for the upload URL. + * + * @return the URI for uploading data. + */ + protected URI createUploadRedirectionURL(UriInfo uriInfo, Enum uploadOperation) { + UriBuilder uriBuilder = uriInfo.getRequestUriBuilder(); + uriBuilder = uriBuilder.replaceQueryParam(OperationParam.NAME, uploadOperation). + queryParam(DataParam.NAME, Boolean.TRUE); + return uriBuilder.build(null); + } + + + /** + * Binding to handle PUT requests. + * + * @param is the inputstream for the request payload. + * @param user the principal of the user making the request. + * @param uriInfo the of the request. + * @param path the path for operation. + * @param op the HttpFS operation of the request. + * @param params the HttpFS parameters of the request. * * @return the request response. * * @throws IOException thrown if an IO error occurred. Thrown exceptions are * handled by {@link HttpFSExceptionProvider}. - * @throws FileSystemAccessException thrown if a FileSystemAccess releated error occurred. Thrown - * exceptions are handled by {@link HttpFSExceptionProvider}. + * @throws FileSystemAccessException thrown if a FileSystemAccess releated + * error occurred. Thrown exceptions are handled by + * {@link HttpFSExceptionProvider}. */ @PUT @Path("{path:.*}") @Consumes({"*/*"}) @Produces({MediaType.APPLICATION_JSON}) public Response put(InputStream is, - @Context Principal user, - @Context UriInfo uriInfo, - @PathParam("path") FsPathParam path, - @QueryParam(PutOpParam.NAME) PutOpParam op, - @QueryParam(ToPathParam.NAME) @DefaultValue(ToPathParam.DEFAULT) ToPathParam toPath, - @QueryParam(OwnerParam.NAME) @DefaultValue(OwnerParam.DEFAULT) OwnerParam owner, - @QueryParam(GroupParam.NAME) @DefaultValue(GroupParam.DEFAULT) GroupParam group, - @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT) OverwriteParam override, - @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT) BlockSizeParam blockSize, - @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT) - PermissionParam permission, - @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT) - ReplicationParam replication, - @QueryParam(ModifiedTimeParam.NAME) @DefaultValue(ModifiedTimeParam.DEFAULT) - ModifiedTimeParam modifiedTime, - @QueryParam(AccessTimeParam.NAME) @DefaultValue(AccessTimeParam.DEFAULT) - AccessTimeParam accessTime, - @QueryParam(DataParam.NAME) @DefaultValue(DataParam.DEFAULT) DataParam hasData, - @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) DoAsParam doAs) + @Context Principal user, + @Context UriInfo uriInfo, + @PathParam("path") String path, + @QueryParam(OperationParam.NAME) OperationParam op, + @Context Parameters params) throws IOException, FileSystemAccessException { - Response response = null; - if (op == null) { - throw new UnsupportedOperationException(MessageFormat.format("Missing [{0}] parameter", PutOpParam.NAME)); - } - path.makeAbsolute(); + Response response; + path = makeAbsolute(path); MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); + String doAs = params.get(DoAsParam.NAME, DoAsParam.class); switch (op.value()) { case CREATE: { - if (!hasData.value()) { + boolean hasData = params.get(DataParam.NAME, DataParam.class); + if (!hasData) { response = Response.temporaryRedirect( - createUploadRedirectionURL(uriInfo, HttpFSFileSystem.PutOpValues.CREATE)).build(); + createUploadRedirectionURL(uriInfo, + HttpFSFileSystem.Operation.CREATE)).build(); } else { - FSOperations.FSCreate - command = new FSOperations.FSCreate(is, path.value(), permission.value(), override.value(), - replication.value(), blockSize.value()); - fsExecute(user, doAs.value(), command); - AUDIT_LOG.info("[{}] permission [{}] override [{}] replication [{}] blockSize [{}]", - new Object[]{path, permission, override, replication, blockSize}); + String permission = params.get(PermissionParam.NAME, + PermissionParam.class); + boolean override = params.get(OverwriteParam.NAME, + OverwriteParam.class); + short replication = params.get(ReplicationParam.NAME, + ReplicationParam.class); + long blockSize = params.get(BlockSizeParam.NAME, + BlockSizeParam.class); + FSOperations.FSCreate command = + new FSOperations.FSCreate(is, path, permission, override, + replication, blockSize); + fsExecute(user, doAs, command); + AUDIT_LOG.info( + "[{}] permission [{}] override [{}] replication [{}] blockSize [{}]", + new Object[]{path, permission, override, replication, blockSize}); response = Response.status(Response.Status.CREATED).build(); } break; } case MKDIRS: { - FSOperations.FSMkdirs command = new FSOperations.FSMkdirs(path.value(), permission.value()); - JSONObject json = fsExecute(user, doAs.value(), command); - AUDIT_LOG.info("[{}] permission [{}]", path, permission.value()); + String permission = params.get(PermissionParam.NAME, + PermissionParam.class); + FSOperations.FSMkdirs command = + new FSOperations.FSMkdirs(path, permission); + JSONObject json = fsExecute(user, doAs, command); + AUDIT_LOG.info("[{}] permission [{}]", path, permission); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case RENAME: { - FSOperations.FSRename command = new FSOperations.FSRename(path.value(), toPath.value()); - JSONObject json = fsExecute(user, doAs.value(), command); + String toPath = params.get(DestinationParam.NAME, DestinationParam.class); + FSOperations.FSRename command = + new FSOperations.FSRename(path, toPath); + JSONObject json = fsExecute(user, doAs, command); AUDIT_LOG.info("[{}] to [{}]", path, toPath); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case SETOWNER: { - FSOperations.FSSetOwner command = new FSOperations.FSSetOwner(path.value(), owner.value(), group.value()); - fsExecute(user, doAs.value(), command); - AUDIT_LOG.info("[{}] to (O/G)[{}]", path, owner.value() + ":" + group.value()); + String owner = params.get(OwnerParam.NAME, OwnerParam.class); + String group = params.get(GroupParam.NAME, GroupParam.class); + FSOperations.FSSetOwner command = + new FSOperations.FSSetOwner(path, owner, group); + fsExecute(user, doAs, command); + AUDIT_LOG.info("[{}] to (O/G)[{}]", path, owner + ":" + group); response = Response.ok().build(); break; } case SETPERMISSION: { - FSOperations.FSSetPermission command = new FSOperations.FSSetPermission(path.value(), permission.value()); - fsExecute(user, doAs.value(), command); - AUDIT_LOG.info("[{}] to [{}]", path, permission.value()); + String permission = params.get(PermissionParam.NAME, + PermissionParam.class); + FSOperations.FSSetPermission command = + new FSOperations.FSSetPermission(path, permission); + fsExecute(user, doAs, command); + AUDIT_LOG.info("[{}] to [{}]", path, permission); response = Response.ok().build(); break; } case SETREPLICATION: { - FSOperations.FSSetReplication command = new FSOperations.FSSetReplication(path.value(), replication.value()); - JSONObject json = fsExecute(user, doAs.value(), command); - AUDIT_LOG.info("[{}] to [{}]", path, replication.value()); + short replication = params.get(ReplicationParam.NAME, + ReplicationParam.class); + FSOperations.FSSetReplication command = + new FSOperations.FSSetReplication(path, replication); + JSONObject json = fsExecute(user, doAs, command); + AUDIT_LOG.info("[{}] to [{}]", path, replication); response = Response.ok(json).build(); break; } case SETTIMES: { - FSOperations.FSSetTimes - command = new FSOperations.FSSetTimes(path.value(), modifiedTime.value(), accessTime.value()); - fsExecute(user, doAs.value(), command); - AUDIT_LOG.info("[{}] to (M/A)[{}]", path, modifiedTime.value() + ":" + accessTime.value()); + long modifiedTime = params.get(ModifiedTimeParam.NAME, + ModifiedTimeParam.class); + long accessTime = params.get(AccessTimeParam.NAME, + AccessTimeParam.class); + FSOperations.FSSetTimes command = + new FSOperations.FSSetTimes(path, modifiedTime, accessTime); + fsExecute(user, doAs, command); + AUDIT_LOG.info("[{}] to (M/A)[{}]", path, + modifiedTime + ":" + accessTime); response = Response.ok().build(); break; } - case RENEWDELEGATIONTOKEN: { - response = Response.status(Response.Status.BAD_REQUEST).build(); - break; - } - case CANCELDELEGATIONTOKEN: { - response = Response.status(Response.Status.BAD_REQUEST).build(); - break; - } - } - return response; - } - - /** - * Binding to handle all OPST requests, supported operations are - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PostOpValues}. - * - * @param is request input stream, used only for - * @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PostOpValues#APPEND} operations. - * @param user principal making the request. - * @param uriInfo the request uriInfo. - * @param path path for the POST request. - * @param op POST operation, default is @link org.apache.hadoop.fs.http.client.HttpFSFileSystem.PostOpValues#APPEND}. - * @param hasData indicates if the append request is uploading data or not (just getting the handle). - * @param doAs user being impersonated, defualt value is none. It can be used - * only if the current user is a HttpFSServer proxyuser. - * - * @return the request response. - * - * @throws IOException thrown if an IO error occurred. Thrown exceptions are - * handled by {@link HttpFSExceptionProvider}. - * @throws FileSystemAccessException thrown if a FileSystemAccess releated error occurred. Thrown - * exceptions are handled by {@link HttpFSExceptionProvider}. - */ - @POST - @Path("{path:.*}") - @Consumes({"*/*"}) - @Produces({MediaType.APPLICATION_JSON}) - public Response post(InputStream is, - @Context Principal user, - @Context UriInfo uriInfo, - @PathParam("path") FsPathParam path, - @QueryParam(PostOpParam.NAME) PostOpParam op, - @QueryParam(DataParam.NAME) @DefaultValue(DataParam.DEFAULT) DataParam hasData, - @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) DoAsParam doAs) - throws IOException, FileSystemAccessException { - Response response = null; - if (op == null) { - throw new UnsupportedOperationException(MessageFormat.format("Missing [{0}] parameter", PostOpParam.NAME)); - } - path.makeAbsolute(); - MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); - switch (op.value()) { - case APPEND: { - if (!hasData.value()) { - response = Response.temporaryRedirect( - createUploadRedirectionURL(uriInfo, HttpFSFileSystem.PostOpValues.APPEND)).build(); - } else { - FSOperations.FSAppend command = new FSOperations.FSAppend(is, path.value()); - fsExecute(user, doAs.value(), command); - AUDIT_LOG.info("[{}]", path); - response = Response.ok().type(MediaType.APPLICATION_JSON).build(); - } - break; + default: { + throw new IOException( + MessageFormat.format("Invalid HTTP PUT operation [{0}]", + op.value())); } } return response; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/BooleanParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/BooleanParam.java index 7bc3a14757..e4e6355063 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/BooleanParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/BooleanParam.java @@ -22,15 +22,14 @@ public abstract class BooleanParam extends Param { - public BooleanParam(String name, String str) { - value = parseParam(name, str); + public BooleanParam(String name, Boolean defaultValue) { + super(name, defaultValue); } protected Boolean parse(String str) throws Exception { if (str.equalsIgnoreCase("true")) { return true; - } - if (str.equalsIgnoreCase("false")) { + } else if (str.equalsIgnoreCase("false")) { return false; } throw new IllegalArgumentException(MessageFormat.format("Invalid value [{0}], must be a boolean", str)); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ByteParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ByteParam.java index aa9408f32e..96b46c4313 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ByteParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ByteParam.java @@ -20,8 +20,8 @@ public abstract class ByteParam extends Param { - public ByteParam(String name, String str) { - value = parseParam(name, str); + public ByteParam(String name, Byte defaultValue) { + super(name, defaultValue); } protected Byte parse(String str) throws Exception { diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/EnumParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/EnumParam.java index ff86406e4a..f605bd2220 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/EnumParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/EnumParam.java @@ -25,9 +25,9 @@ public abstract class EnumParam> extends Param { Class klass; - public EnumParam(String label, String str, Class e) { + public EnumParam(String name, Class e, E defaultValue) { + super(name, defaultValue); klass = e; - value = parseParam(label, str); } protected E parse(String str) throws Exception { diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/IntegerParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/IntegerParam.java index 6eddaa2e5f..7c0f0813c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/IntegerParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/IntegerParam.java @@ -20,8 +20,8 @@ public abstract class IntegerParam extends Param { - public IntegerParam(String name, String str) { - value = parseParam(name, str); + public IntegerParam(String name, Integer defaultValue) { + super(name, defaultValue); } protected Integer parse(String str) throws Exception { diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/LongParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/LongParam.java index 354a550d7b..ec601bb2ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/LongParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/LongParam.java @@ -20,8 +20,8 @@ public abstract class LongParam extends Param { - public LongParam(String name, String str) { - value = parseParam(name, str); + public LongParam(String name, Long defaultValue) { + super(name, defaultValue); } protected Long parse(String str) throws Exception { diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Param.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Param.java index 68a41d5151..62af4818f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Param.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Param.java @@ -23,32 +23,39 @@ import java.text.MessageFormat; public abstract class Param { + private String name; protected T value; - public T parseParam(String name, String str) { - Check.notNull(name, "name"); + public Param(String name, T defaultValue) { + this.name = name; + this.value = defaultValue; + } + + public String getName() { + return name; + } + + public T parseParam(String str) { try { - return (str != null && str.trim().length() > 0) ? parse(str) : null; + value = (str != null && str.trim().length() > 0) ? parse(str) : value; } catch (Exception ex) { throw new IllegalArgumentException( MessageFormat.format("Parameter [{0}], invalid value [{1}], value must be [{2}]", name, str, getDomain())); } + return value; } public T value() { return value; } - protected void setValue(T value) { - this.value = value; - } - protected abstract String getDomain(); protected abstract T parse(String str) throws Exception; public String toString() { - return value.toString(); + return (value != null) ? value.toString() : "NULL"; } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestLongParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java similarity index 52% rename from hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestLongParam.java rename to hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java index 1a7ddd8d35..b5ec214d7a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestLongParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java @@ -15,33 +15,37 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.lib.wsrs; -import junit.framework.Assert; -import org.junit.Test; +import java.util.Map; -public class TestLongParam { +/** + * Class that contains all parsed JAX-RS parameters. + *

+ * Instances are created by the {@link ParametersProvider} class. + */ +public class Parameters { + private Map> params; - @Test - public void param() throws Exception { - LongParam param = new LongParam("p", "1") { - }; - Assert.assertEquals(param.getDomain(), "a long"); - Assert.assertEquals(param.value(), new Long(1)); - Assert.assertEquals(param.toString(), "1"); - param = new LongParam("p", null) { - }; - Assert.assertEquals(param.value(), null); - param = new LongParam("p", "") { - }; - Assert.assertEquals(param.value(), null); + /** + * Constructor that receives the request parsed parameters. + * + * @param params the request parsed parameters. + */ + public Parameters(Map> params) { + this.params = params; } - @Test(expected = IllegalArgumentException.class) - public void invalid1() throws Exception { - new LongParam("p", "x") { - }; + /** + * Returns the value of a request parsed parameter. + * + * @param name parameter name. + * @param klass class of the parameter, used for value casting. + * @return the value of the parameter. + */ + @SuppressWarnings("unchecked") + public > V get(String name, Class klass) { + return ((T)params.get(name)).value(); } - + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ParametersProvider.java new file mode 100644 index 0000000000..3d41d991ad --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ParametersProvider.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.lib.wsrs; + +import com.sun.jersey.api.core.HttpContext; +import com.sun.jersey.core.spi.component.ComponentContext; +import com.sun.jersey.core.spi.component.ComponentScope; +import com.sun.jersey.server.impl.inject.AbstractHttpContextInjectable; +import com.sun.jersey.spi.inject.Injectable; +import com.sun.jersey.spi.inject.InjectableProvider; + +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MultivaluedMap; +import java.lang.reflect.Type; +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.Map; + +/** + * Jersey provider that parses the request parameters based on the + * given parameter definition. + */ +public class ParametersProvider + extends AbstractHttpContextInjectable + implements InjectableProvider { + + private String driverParam; + private Class enumClass; + private Map>[]> paramsDef; + + public ParametersProvider(String driverParam, Class enumClass, + Map>[]> paramsDef) { + this.driverParam = driverParam; + this.enumClass = enumClass; + this.paramsDef = paramsDef; + } + + @Override + @SuppressWarnings("unchecked") + public Parameters getValue(HttpContext httpContext) { + Map> map = new HashMap>(); + MultivaluedMap queryString = + httpContext.getRequest().getQueryParameters(); + String str = queryString.getFirst(driverParam); + if (str == null) { + throw new IllegalArgumentException( + MessageFormat.format("Missing Operation parameter [{0}]", + driverParam)); + } + Enum op; + try { + op = Enum.valueOf(enumClass, str.toUpperCase()); + } catch (IllegalArgumentException ex) { + throw new IllegalArgumentException( + MessageFormat.format("Invalid Operation [{0}]", str)); + } + if (!paramsDef.containsKey(op)) { + throw new IllegalArgumentException( + MessageFormat.format("Unsupported Operation [{0}]", op)); + } + for (Class> paramClass : paramsDef.get(op)) { + Param param; + try { + param = paramClass.newInstance(); + } catch (Exception ex) { + throw new UnsupportedOperationException( + MessageFormat.format( + "Param class [{0}] does not have default constructor", + paramClass.getName())); + } + try { + param.parseParam(queryString.getFirst(param.getName())); + } + catch (Exception ex) { + throw new IllegalArgumentException(ex.toString(), ex); + } + map.put(param.getName(), param); + } + return new Parameters(map); + } + + @Override + public ComponentScope getScope() { + return ComponentScope.PerRequest; + } + + @Override + public Injectable getInjectable(ComponentContext componentContext, Context context, Type type) { + return (type.equals(Parameters.class)) ? this : null; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ShortParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ShortParam.java index a3995baa61..cc75a86062 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ShortParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ShortParam.java @@ -20,8 +20,8 @@ public abstract class ShortParam extends Param { - public ShortParam(String name, String str) { - value = parseParam(name, str); + public ShortParam(String name, Short defaultValue) { + super(name, defaultValue); } protected Short parse(String str) throws Exception { diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/StringParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/StringParam.java index 4b3a9274fe..79e633697f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/StringParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/StringParam.java @@ -15,42 +15,38 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.lib.wsrs; -import org.apache.hadoop.lib.util.Check; - import java.text.MessageFormat; import java.util.regex.Pattern; public abstract class StringParam extends Param { private Pattern pattern; - public StringParam(String name, String str) { - this(name, str, null); + public StringParam(String name, String defaultValue) { + this(name, defaultValue, null); } - public StringParam(String name, String str, Pattern pattern) { + public StringParam(String name, String defaultValue, Pattern pattern) { + super(name, defaultValue); this.pattern = pattern; - value = parseParam(name, str); + parseParam(defaultValue); } - public String parseParam(String name, String str) { - String ret = null; - Check.notNull(name, "name"); + public String parseParam(String str) { try { if (str != null) { str = str.trim(); if (str.length() > 0) { - return parse(str); + value = parse(str); } } } catch (Exception ex) { throw new IllegalArgumentException( MessageFormat.format("Parameter [{0}], invalid value [{1}], value must be [{2}]", - name, str, getDomain())); + getName(), str, getDomain())); } - return ret; + return value; } protected String parse(String str) throws Exception { diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java index a55d5e2a46..e2f8b842f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java @@ -475,6 +475,7 @@ public static Collection operations() { ops[i] = new Object[]{Operation.values()[i]}; } return Arrays.asList(ops); +// return Arrays.asList(new Object[][]{ new Object[]{Operation.CREATE}}); } private Operation operation; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestCheckUploadContentTypeFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestCheckUploadContentTypeFilter.java index 2596be9754..9996e0bea0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestCheckUploadContentTypeFilter.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestCheckUploadContentTypeFilter.java @@ -31,34 +31,34 @@ public class TestCheckUploadContentTypeFilter { @Test public void putUpload() throws Exception { - test("PUT", HttpFSFileSystem.PutOpValues.CREATE.toString(), "application/octet-stream", true, false); + test("PUT", HttpFSFileSystem.Operation.CREATE.toString(), "application/octet-stream", true, false); } @Test public void postUpload() throws Exception { - test("POST", HttpFSFileSystem.PostOpValues.APPEND.toString(), "APPLICATION/OCTET-STREAM", true, false); + test("POST", HttpFSFileSystem.Operation.APPEND.toString(), "APPLICATION/OCTET-STREAM", true, false); } @Test public void putUploadWrong() throws Exception { - test("PUT", HttpFSFileSystem.PutOpValues.CREATE.toString(), "plain/text", false, false); - test("PUT", HttpFSFileSystem.PutOpValues.CREATE.toString(), "plain/text", true, true); + test("PUT", HttpFSFileSystem.Operation.CREATE.toString(), "plain/text", false, false); + test("PUT", HttpFSFileSystem.Operation.CREATE.toString(), "plain/text", true, true); } @Test public void postUploadWrong() throws Exception { - test("POST", HttpFSFileSystem.PostOpValues.APPEND.toString(), "plain/text", false, false); - test("POST", HttpFSFileSystem.PostOpValues.APPEND.toString(), "plain/text", true, true); + test("POST", HttpFSFileSystem.Operation.APPEND.toString(), "plain/text", false, false); + test("POST", HttpFSFileSystem.Operation.APPEND.toString(), "plain/text", true, true); } @Test public void getOther() throws Exception { - test("GET", HttpFSFileSystem.GetOpValues.GETHOMEDIRECTORY.toString(), "plain/text", false, false); + test("GET", HttpFSFileSystem.Operation.GETHOMEDIRECTORY.toString(), "plain/text", false, false); } @Test public void putOther() throws Exception { - test("PUT", HttpFSFileSystem.PutOpValues.MKDIRS.toString(), "plain/text", false, false); + test("PUT", HttpFSFileSystem.Operation.MKDIRS.toString(), "plain/text", false, false); } private void test(String method, String operation, String contentType, @@ -68,7 +68,7 @@ private void test(String method, String operation, String contentType, Mockito.reset(request); Mockito.when(request.getMethod()).thenReturn(method); Mockito.when(request.getParameter(HttpFSFileSystem.OP_PARAM)).thenReturn(operation); - Mockito.when(request.getParameter(HttpFSParams.DataParam.NAME)). + Mockito.when(request.getParameter(HttpFSParametersProvider.DataParam.NAME)). thenReturn(Boolean.toString(upload)); Mockito.when(request.getContentType()).thenReturn(contentType); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestBooleanParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestBooleanParam.java deleted file mode 100644 index b1b140d7cd..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestBooleanParam.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.lib.wsrs; - -import junit.framework.Assert; -import org.junit.Test; - -public class TestBooleanParam { - - @Test - public void param() throws Exception { - BooleanParam param = new BooleanParam("p", "true") { - }; - Assert.assertEquals(param.getDomain(), "a boolean"); - Assert.assertEquals(param.value(), Boolean.TRUE); - Assert.assertEquals(param.toString(), "true"); - param = new BooleanParam("p", "false") { - }; - Assert.assertEquals(param.value(), Boolean.FALSE); - param = new BooleanParam("p", null) { - }; - Assert.assertEquals(param.value(), null); - param = new BooleanParam("p", "") { - }; - Assert.assertEquals(param.value(), null); - } - - @Test(expected = IllegalArgumentException.class) - public void invalid() throws Exception { - new BooleanParam("p", "x") { - }; - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestByteParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestByteParam.java deleted file mode 100644 index 6b1a5ef64c..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestByteParam.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.lib.wsrs; - - -import junit.framework.Assert; -import org.junit.Test; - -public class TestByteParam { - - @Test - public void param() throws Exception { - ByteParam param = new ByteParam("p", "1") { - }; - Assert.assertEquals(param.getDomain(), "a byte"); - Assert.assertEquals(param.value(), new Byte((byte) 1)); - Assert.assertEquals(param.toString(), "1"); - param = new ByteParam("p", null) { - }; - Assert.assertEquals(param.value(), null); - param = new ByteParam("p", "") { - }; - Assert.assertEquals(param.value(), null); - } - - @Test(expected = IllegalArgumentException.class) - public void invalid1() throws Exception { - new ByteParam("p", "x") { - }; - } - - @Test(expected = IllegalArgumentException.class) - public void invalid2() throws Exception { - new ByteParam("p", "256") { - }; - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestEnumParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestEnumParam.java deleted file mode 100644 index bb37f75f37..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestEnumParam.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.lib.wsrs; - - -import junit.framework.Assert; -import org.junit.Test; - -public class TestEnumParam { - - public static enum ENUM { - FOO, BAR - } - - @Test - public void param() throws Exception { - EnumParam param = new EnumParam("p", "FOO", ENUM.class) { - }; - Assert.assertEquals(param.getDomain(), "FOO,BAR"); - Assert.assertEquals(param.value(), ENUM.FOO); - Assert.assertEquals(param.toString(), "FOO"); - param = new EnumParam("p", null, ENUM.class) { - }; - Assert.assertEquals(param.value(), null); - param = new EnumParam("p", "", ENUM.class) { - }; - Assert.assertEquals(param.value(), null); - } - - @Test(expected = IllegalArgumentException.class) - public void invalid1() throws Exception { - new EnumParam("p", "x", ENUM.class) { - }; - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestIntegerParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestIntegerParam.java deleted file mode 100644 index 634dbe7c2a..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestIntegerParam.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.lib.wsrs; - -import junit.framework.Assert; -import org.junit.Test; - -public class TestIntegerParam { - - @Test - public void param() throws Exception { - IntegerParam param = new IntegerParam("p", "1") { - }; - Assert.assertEquals(param.getDomain(), "an integer"); - Assert.assertEquals(param.value(), new Integer(1)); - Assert.assertEquals(param.toString(), "1"); - param = new IntegerParam("p", null) { - }; - Assert.assertEquals(param.value(), null); - param = new IntegerParam("p", "") { - }; - Assert.assertEquals(param.value(), null); - } - - @Test(expected = IllegalArgumentException.class) - public void invalid1() throws Exception { - new IntegerParam("p", "x") { - }; - } - - @Test(expected = IllegalArgumentException.class) - public void invalid2() throws Exception { - new IntegerParam("p", "" + Long.MAX_VALUE) { - }; - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestParam.java new file mode 100644 index 0000000000..ed79c86e7d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestParam.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.lib.wsrs; + +import junit.framework.Assert; +import org.junit.Test; + +import java.util.regex.Pattern; + +public class TestParam { + + private void test(Param param, String name, + String domain, T defaultValue, T validValue, + String invalidStrValue, String outOfRangeValue) throws Exception { + + Assert.assertEquals(name, param.getName()); + Assert.assertEquals(domain, param.getDomain()); + Assert.assertEquals(defaultValue, param.value()); + Assert.assertEquals(defaultValue, param.parseParam("")); + Assert.assertEquals(defaultValue, param.parseParam(null)); + Assert.assertEquals(validValue, param.parseParam(validValue.toString())); + if (invalidStrValue != null) { + try { + param.parseParam(invalidStrValue); + Assert.fail(); + } catch (IllegalArgumentException ex) { + //NOP + } catch (Exception ex) { + Assert.fail(); + } + } + if (outOfRangeValue != null) { + try { + param.parseParam(outOfRangeValue); + Assert.fail(); + } catch (IllegalArgumentException ex) { + //NOP + } catch (Exception ex) { + Assert.fail(); + } + } + } + + @Test + public void testBoolean() throws Exception { + Param param = new BooleanParam("b", false) { + }; + test(param, "b", "a boolean", false, true, "x", null); + } + + @Test + public void testByte() throws Exception { + Param param = new ByteParam("B", (byte) 1) { + }; + test(param, "B", "a byte", (byte) 1, (byte) 2, "x", "256"); + } + + @Test + public void testShort() throws Exception { + Param param = new ShortParam("S", (short) 1) { + }; + test(param, "S", "a short", (short) 1, (short) 2, "x", + "" + ((int)Short.MAX_VALUE + 1)); + } + + @Test + public void testInteger() throws Exception { + Param param = new IntegerParam("I", 1) { + }; + test(param, "I", "an integer", 1, 2, "x", "" + ((long)Integer.MAX_VALUE + 1)); + } + + @Test + public void testLong() throws Exception { + Param param = new LongParam("L", 1L) { + }; + test(param, "L", "a long", 1L, 2L, "x", null); + } + + public static enum ENUM { + FOO, BAR + } + + @Test + public void testEnum() throws Exception { + EnumParam param = new EnumParam("e", ENUM.class, ENUM.FOO) { + }; + test(param, "e", "FOO,BAR", ENUM.FOO, ENUM.BAR, "x", null); + } + + @Test + public void testString() throws Exception { + Param param = new StringParam("s", "foo") { + }; + test(param, "s", "a string", "foo", "bar", null, null); + } + + @Test + public void testRegEx() throws Exception { + Param param = new StringParam("r", "aa", Pattern.compile("..")) { + }; + test(param, "r", "..", "aa", "bb", "c", null); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestShortParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestShortParam.java deleted file mode 100644 index b37bddffe4..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestShortParam.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.lib.wsrs; - - -import junit.framework.Assert; -import org.junit.Test; - -public class TestShortParam { - - @Test - public void param() throws Exception { - ShortParam param = new ShortParam("p", "1") { - }; - Assert.assertEquals(param.getDomain(), "a short"); - Assert.assertEquals(param.value(), new Short((short) 1)); - Assert.assertEquals(param.toString(), "1"); - param = new ShortParam("p", null) { - }; - Assert.assertEquals(param.value(), null); - param = new ShortParam("p", "") { - }; - Assert.assertEquals(param.value(), null); - } - - @Test(expected = IllegalArgumentException.class) - public void invalid1() throws Exception { - new ShortParam("p", "x") { - }; - } - - @Test(expected = IllegalArgumentException.class) - public void invalid2() throws Exception { - new ShortParam("p", "" + Integer.MAX_VALUE) { - }; - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestStringParam.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestStringParam.java deleted file mode 100644 index feb489e043..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestStringParam.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.lib.wsrs; - - -import junit.framework.Assert; -import org.junit.Test; - -import java.util.regex.Pattern; - -public class TestStringParam { - - @Test - public void param() throws Exception { - StringParam param = new StringParam("p", "s") { - }; - Assert.assertEquals(param.getDomain(), "a string"); - Assert.assertEquals(param.value(), "s"); - Assert.assertEquals(param.toString(), "s"); - param = new StringParam("p", null) { - }; - Assert.assertEquals(param.value(), null); - param = new StringParam("p", "") { - }; - Assert.assertEquals(param.value(), null); - - param.setValue("S"); - Assert.assertEquals(param.value(), "S"); - } - - @Test - public void paramRegEx() throws Exception { - StringParam param = new StringParam("p", "Aaa", Pattern.compile("A.*")) { - }; - Assert.assertEquals(param.getDomain(), "A.*"); - Assert.assertEquals(param.value(), "Aaa"); - Assert.assertEquals(param.toString(), "Aaa"); - param = new StringParam("p", null) { - }; - Assert.assertEquals(param.value(), null); - } - - @Test(expected = IllegalArgumentException.class) - public void paramInvalidRegEx() throws Exception { - new StringParam("p", "Baa", Pattern.compile("A.*")) { - }; - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/resources/httpfs-log4j.properties b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/resources/httpfs-log4j.properties new file mode 100644 index 0000000000..75175124c5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/resources/httpfs-log4j.properties @@ -0,0 +1,22 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#log4j.appender.test=org.apache.log4j.varia.NullAppender +#log4j.appender.test=org.apache.log4j.ConsoleAppender +log4j.appender.test=org.apache.log4j.FileAppender +log4j.appender.test.File=${test.dir}/test.log +log4j.appender.test.Append=true +log4j.appender.test.layout=org.apache.log4j.PatternLayout +log4j.appender.test.layout.ConversionPattern=%d{ISO8601} %5p %20c{1}: %4L - %m%n +log4j.rootLogger=ALL, test + diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index bf58013749..3b26916ce5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -246,6 +246,8 @@ Branch-2 ( Unreleased changes ) HDFS-3535. Audit logging should log denied accesses. (Andy Isaacson via eli) + HDFS-3481. Refactor HttpFS handling of JAX-RS query string parameters (tucu) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log From 530d49ae1536e790ca8fa0853ecdf8065b865124 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 27 Jun 2012 16:09:48 +0000 Subject: [PATCH 85/91] Amendment to previous commit, correct JIRA is HDFS-3113 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1354602 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3b26916ce5..8a7450ada6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -246,7 +246,7 @@ Branch-2 ( Unreleased changes ) HDFS-3535. Audit logging should log denied accesses. (Andy Isaacson via eli) - HDFS-3481. Refactor HttpFS handling of JAX-RS query string parameters (tucu) + HDFS-3113. Refactor HttpFS handling of JAX-RS query string parameters (tucu) OPTIMIZATIONS From c1fe5d5c3051c3da766c5c68a625cecb9cd428af Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 27 Jun 2012 16:20:47 +0000 Subject: [PATCH 86/91] Re-amendment to previous commit JIRA was HDFS-3481 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1354611 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8a7450ada6..3b26916ce5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -246,7 +246,7 @@ Branch-2 ( Unreleased changes ) HDFS-3535. Audit logging should log denied accesses. (Andy Isaacson via eli) - HDFS-3113. Refactor HttpFS handling of JAX-RS query string parameters (tucu) + HDFS-3481. Refactor HttpFS handling of JAX-RS query string parameters (tucu) OPTIMIZATIONS From a2c450d952071b2868cda5a16b0760dffd1cda3e Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 27 Jun 2012 18:29:42 +0000 Subject: [PATCH 87/91] Reverting MAPREDUCE-4346 r1353757 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1354656 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 - ...obClient.java => TestJobClientGetJob.java} | 22 +-------- .../org/apache/hadoop/mapred/JobClient.java | 47 +++++-------------- .../org/apache/hadoop/mapreduce/Cluster.java | 15 +----- 4 files changed, 16 insertions(+), 70 deletions(-) rename hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/{TestJobClient.java => TestJobClientGetJob.java} (71%) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d80e42955a..e82154515d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -129,8 +129,6 @@ Branch-2 ( Unreleased changes ) NEW FEATURES - MAPREDUCE-4355. Add JobStatus getJobStatus(JobID) to JobClient. (kkambatl via tucu) - IMPROVEMENTS MAPREDUCE-4146. Support limits on task status string length and number of diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java similarity index 71% rename from hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClient.java rename to hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java index 87afeb326c..f929e4f4a8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClient.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java @@ -19,7 +19,6 @@ package org.apache.hadoop.mapred; import static junit.framework.Assert.assertNotNull; -import static org.junit.Assert.assertEquals; import java.io.IOException; @@ -29,7 +28,7 @@ import org.apache.hadoop.fs.Path; import org.junit.Test; -public class TestJobClient { +public class TestJobClientGetJob { private static Path TEST_ROOT_DIR = new Path(System.getProperty("test.build.data","/tmp")); @@ -46,7 +45,7 @@ private Path createTempFile(String filename, String contents) @SuppressWarnings("deprecation") @Test - public void testGetRunningJob() throws Exception { + public void testGetRunningJobFromJobClient() throws Exception { JobConf conf = new JobConf(); conf.set("mapreduce.framework.name", "local"); FileInputFormat.addInputPath(conf, createTempFile("in", "hello")); @@ -61,21 +60,4 @@ public void testGetRunningJob() throws Exception { assertNotNull("New running job", newRunningJob); } - @SuppressWarnings("deprecation") - @Test - public void testGetJobStatus() throws Exception { - JobConf conf = new JobConf(); - conf.set("mapreduce.framework.name", "local"); - FileInputFormat.addInputPath(conf, createTempFile("in", "hello")); - Path outputDir = new Path(TEST_ROOT_DIR, getClass().getSimpleName()); - outputDir.getFileSystem(conf).delete(outputDir, true); - FileOutputFormat.setOutputPath(conf, outputDir); - JobClient jc = new JobClient(conf); - RunningJob runningJob = jc.submitJob(conf); - assertNotNull("Running job", runningJob); - JobID jobid = runningJob.getID(); - JobStatus jobStatus = jc.getJobStatus(jobid); - assertNotNull("New running job", jobStatus); - assertEquals("Equal JobIDs", jobid, jobStatus.getJobID()); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java index 0bc4eaea8f..5d7919dbea 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java @@ -620,15 +620,6 @@ public Job run() throws IOException, ClassNotFoundException, } } - private JobStatus getJobStatusUsingCluster(final JobID jobId) - throws IOException, InterruptedException { - return clientUgi.doAs(new PrivilegedExceptionAction() { - public JobStatus run() throws IOException, InterruptedException { - return JobStatus.downgrade(cluster.getJobStatus(jobId)); - } - }); - } - private Job getJobUsingCluster(final JobID jobid) throws IOException, InterruptedException { return clientUgi.doAs(new PrivilegedExceptionAction() { @@ -637,40 +628,28 @@ public Job run() throws IOException, InterruptedException { } }); } - /** - * Get {@link JobStatus} of a job. Returns null if the id does not correspond - * to any known job. + * Get an {@link RunningJob} object to track an ongoing job. Returns + * null if the id does not correspond to any known job. * - * @param jobid - * the jobid of the job. - * @return the {@link JobStatus} object to retrieve the job stats, null if the + * @param jobid the jobid of the job. + * @return the {@link RunningJob} handle to track the job, null if the * jobid doesn't correspond to any known job. * @throws IOException */ - public JobStatus getJobStatus(JobID jobId) throws IOException { + public RunningJob getJob(final JobID jobid) throws IOException { try { - return getJobStatusUsingCluster(jobId); + + Job job = getJobUsingCluster(jobid); + if (job != null) { + JobStatus status = JobStatus.downgrade(job.getStatus()); + if (status != null) { + return new NetworkedJob(status, cluster); + } + } } catch (InterruptedException ie) { throw new IOException(ie); } - } - - /** - * Get an {@link RunningJob} object to track an ongoing job. Returns null if - * the id does not correspond to any known job. - * - * @param jobid - * the jobid of the job. - * @return the {@link RunningJob} handle to track the job, null if the - * jobid doesn't correspond to any known job. - * @throws IOException - */ - public RunningJob getJob(JobID jobId) throws IOException { - JobStatus status = getJobStatus(jobId); - if (status != null) { - return new NetworkedJob(status, cluster); - } return null; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java index a3aee69550..e456a7afa8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java @@ -172,19 +172,6 @@ public FileSystem run() throws IOException, InterruptedException { return fs; } - /** - * Get JobStatus corresponding to jobId. - * - * @param jobId - * @return object of {@link JobStatus} - * @throws IOException - * @throws InterruptedException - */ - public JobStatus getJobStatus(JobID jobId) throws IOException, - InterruptedException { - return client.getJobStatus(jobId); - } - /** * Get job corresponding to jobid. * @@ -194,7 +181,7 @@ public JobStatus getJobStatus(JobID jobId) throws IOException, * @throws InterruptedException */ public Job getJob(JobID jobId) throws IOException, InterruptedException { - JobStatus status = getJobStatus(jobId); + JobStatus status = client.getJobStatus(jobId); if (status != null) { return Job.getInstance(this, status, new JobConf(status.getJobFile())); } From 97a583f709e35d2c638d9a4a562723eb22dd9934 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 27 Jun 2012 19:32:21 +0000 Subject: [PATCH 88/91] HADOOP-8059. Add javadoc to InterfaceAudience and InterfaceStability. Contributed by Brandon Li. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1354684 13f79535-47bb-0310-9956-ffa450edef68 --- .../classification/InterfaceAudience.java | 18 ++++++++++++++++++ .../classification/InterfaceStability.java | 13 ++++++++++++- .../hadoop-common/CHANGES.txt | 3 +++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceAudience.java b/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceAudience.java index 019874fb4a..cd62a9469c 100644 --- a/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceAudience.java +++ b/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceAudience.java @@ -21,6 +21,24 @@ /** * Annotation to inform users of a package, class or method's intended audience. + * Currently the audience can be {@link Public}, {@link LimitedPrivate} or + * {@link Private}.
+ * All public classes must have InterfaceAudience annotation.
+ *

    + *
  • Public classes that are not marked with this annotation must be + * considered by default as {@link Private}.
    • + * + *
  • External applications must only use classes that are marked + * {@link Public}. Avoid using non public classes as these classes + * could be removed or change in incompatible ways.
    • + * + *
  • Hadoop projects must only use classes that are marked + * {@link LimitedPrivate} or {@link Public}
    • + * + *
  • Methods may have a different annotation that it is more restrictive + * compared to the audience classification of the class. Example: A class + * might be {@link Public}, but a method may be {@link LimitedPrivate} + *
*/ @InterfaceAudience.Public @InterfaceStability.Evolving diff --git a/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceStability.java b/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceStability.java index 0ebf949822..f78acc296c 100644 --- a/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceStability.java +++ b/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceStability.java @@ -19,9 +19,20 @@ import java.lang.annotation.Documented; +import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; + /** * Annotation to inform users of how much to rely on a particular package, - * class or method not changing over time. + * class or method not changing over time. Currently the stability can be + * {@link Stable}, {@link Evolving} or {@link Unstable}.
+ * + *
  • All classes that are annotated with {@link Public} or + * {@link LimitedPrivate} must have InterfaceStability annotation.
    • + *
  • Classes that are {@link Private} are to be considered unstable unless + * a different InterfaceStability annotation states otherwise.
    • + *
  • Incompatible changes must not be made to classes marked as stable.
    • */ @InterfaceAudience.Public @InterfaceStability.Evolving diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 1fed42d0cb..a9c99aedd5 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -79,6 +79,9 @@ Trunk (unreleased changes) HADOOP-7659. fs -getmerge isn't guaranteed to work well over non-HDFS filesystems (harsh) + HADOOP-8059. Add javadoc to InterfaceAudience and InterfaceStability. + (Brandon Li via suresh) + BUG FIXES HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName. From eec642c68b58f0beaef20b6c73c24070c4b02222 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 27 Jun 2012 22:41:41 +0000 Subject: [PATCH 89/91] HDFS-3572. Cleanup code which inits SPNEGO in HttpServer. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1354767 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/http/HttpServer.java | 20 +++++++++++++++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../documentation/content/xdocs/webhdfs.xml | 2 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++-- .../server/namenode/NameNodeHttpServer.java | 20 +++---------------- .../server/namenode/SecondaryNameNode.java | 19 ++---------------- 6 files changed, 30 insertions(+), 37 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java index ded6870254..2f693b4714 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java @@ -52,7 +52,9 @@ import org.apache.hadoop.jmx.JMXJsonServlet; import org.apache.hadoop.log.LogLevel; import org.apache.hadoop.metrics.MetricsServlet; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.util.ReflectionUtils; import org.mortbay.io.Buffer; @@ -606,6 +608,24 @@ public void addSslListener(InetSocketAddress addr, Configuration sslConf, sslListener.setNeedClientAuth(needCertsAuth); webServer.addConnector(sslListener); } + + protected void initSpnego(Configuration conf, + String usernameConfKey, String keytabConfKey) throws IOException { + Map params = new HashMap(); + String principalInConf = conf.get(usernameConfKey); + if (principalInConf != null && !principalInConf.isEmpty()) { + params.put("kerberos.principal", + SecurityUtil.getServerPrincipal(principalInConf, listener.getHost())); + } + String httpKeytab = conf.get(keytabConfKey); + if (httpKeytab != null && !httpKeytab.isEmpty()) { + params.put("kerberos.keytab", httpKeytab); + } + params.put(AuthenticationFilter.AUTH_TYPE, "kerberos"); + + defineFilter(webAppContext, SPNEGO_FILTER, + AuthenticationFilter.class.getName(), params, null); + } /** * Start the server. Does not wait for the server to start. diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3b26916ce5..a708333c08 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -248,6 +248,8 @@ Branch-2 ( Unreleased changes ) HDFS-3481. Refactor HttpFS handling of JAX-RS query string parameters (tucu) + HDFS-3572. Cleanup code which inits SPNEGO in HttpServer (todd) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/webhdfs.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/webhdfs.xml index 43764ca275..c8e0c627dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/webhdfs.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/webhdfs.xml @@ -152,7 +152,7 @@ dfs.web.authentication.kerberos.principal The HTTP Kerberos principal used by Hadoop-Auth in the HTTP endpoint. The HTTP Kerberos principal MUST start with 'HTTP/' per Kerberos - HTTP SPENGO specification. + HTTP SPNEGO specification. dfs.web.authentication.kerberos.keytab The Kerberos keytab file with the credentials for the diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 51af55850d..146ed8358f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -323,10 +323,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_DATANODE_USER_NAME_KEY = "dfs.datanode.kerberos.principal"; public static final String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file"; public static final String DFS_NAMENODE_USER_NAME_KEY = "dfs.namenode.kerberos.principal"; - public static final String DFS_NAMENODE_INTERNAL_SPENGO_USER_NAME_KEY = "dfs.namenode.kerberos.internal.spnego.principal"; + public static final String DFS_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY = "dfs.namenode.kerberos.internal.spnego.principal"; public static final String DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY = "dfs.secondary.namenode.keytab.file"; public static final String DFS_SECONDARY_NAMENODE_USER_NAME_KEY = "dfs.secondary.namenode.kerberos.principal"; - public static final String DFS_SECONDARY_NAMENODE_INTERNAL_SPENGO_USER_NAME_KEY = "dfs.secondary.namenode.kerberos.internal.spnego.principal"; + public static final String DFS_SECONDARY_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY = "dfs.secondary.namenode.kerberos.internal.spnego.principal"; public static final String DFS_NAMENODE_NAME_CACHE_THRESHOLD_KEY = "dfs.namenode.name.cache.threshold"; public static final int DFS_NAMENODE_NAME_CACHE_THRESHOLD_DEFAULT = 10; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index 2798a83bf8..44b0437d13 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -44,7 +44,6 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authorize.AccessControlList; /** @@ -91,22 +90,9 @@ public void start() throws IOException { { // Add SPNEGO support to NameNode if (UserGroupInformation.isSecurityEnabled()) { - Map params = new HashMap(); - String principalInConf = conf.get( - DFSConfigKeys.DFS_NAMENODE_INTERNAL_SPENGO_USER_NAME_KEY); - if (principalInConf != null && !principalInConf.isEmpty()) { - params.put("kerberos.principal", - SecurityUtil.getServerPrincipal(principalInConf, infoHost)); - String httpKeytab = conf.get(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY); - if (httpKeytab != null && !httpKeytab.isEmpty()) { - params.put("kerberos.keytab", httpKeytab); - } - - params.put(AuthenticationFilter.AUTH_TYPE, "kerberos"); - - defineFilter(webAppContext, SPNEGO_FILTER, - AuthenticationFilter.class.getName(), params, null); - } + initSpnego(conf, + DFSConfigKeys.DFS_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY, + DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY); } if (WebHdfsFileSystem.isEnabled(conf, LOG)) { //add SPNEGO authentication filter for webhdfs diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index 7c02c644da..0d79fe27a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -25,10 +25,8 @@ import java.security.PrivilegedExceptionAction; import java.util.Collection; import java.util.Date; -import java.util.HashMap; import java.util.Iterator; import java.util.List; -import java.util.Map; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -68,7 +66,6 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.util.Daemon; @@ -239,20 +236,8 @@ private void initialize(final Configuration conf, new AccessControlList(conf.get(DFS_ADMIN, " "))) { { if (UserGroupInformation.isSecurityEnabled()) { - Map params = new HashMap(); - String principalInConf = conf.get(DFSConfigKeys.DFS_SECONDARY_NAMENODE_INTERNAL_SPENGO_USER_NAME_KEY); - if (principalInConf != null && !principalInConf.isEmpty()) { - params.put("kerberos.principal", - SecurityUtil.getServerPrincipal(principalInConf, infoSocAddr.getHostName())); - } - String httpKeytab = conf.get(DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY); - if (httpKeytab != null && !httpKeytab.isEmpty()) { - params.put("kerberos.keytab", httpKeytab); - } - params.put(AuthenticationFilter.AUTH_TYPE, "kerberos"); - - defineFilter(webAppContext, SPNEGO_FILTER, AuthenticationFilter.class.getName(), - params, null); + initSpnego(conf, DFSConfigKeys.DFS_SECONDARY_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY, + DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY); } } }; From dd4c39457c0705bc5ce8ebfa86a0b35b7fcd3efb Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Thu, 28 Jun 2012 01:11:22 +0000 Subject: [PATCH 90/91] HDFS-3576. Move the definition of the constant NameNode.DEFAULT_PORT to DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT. Contributed by Brandon Li git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1354790 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/fs/Hdfs.java | 7 ++++--- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 1 + .../apache/hadoop/hdfs/DistributedFileSystem.java | 2 +- .../org/apache/hadoop/hdfs/NameNodeProxies.java | 2 +- .../token/delegation/DelegationTokenSelector.java | 4 ++-- .../hadoop/hdfs/server/namenode/NameNode.java | 6 +++--- .../hadoop/hdfs/tools/NNHAServiceTarget.java | 2 +- .../apache/hadoop/hdfs/TestDFSClientFailover.java | 7 +++---- .../hadoop/hdfs/TestDefaultNameNodePort.java | 12 +++++++----- .../balancer/TestBalancerWithHANameNodes.java | 4 ++-- .../java/org/apache/hadoop/fs/TestFileSystem.java | 15 ++++++++------- 12 files changed, 36 insertions(+), 29 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a708333c08..5cf4a7ddc1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -250,6 +250,9 @@ Branch-2 ( Unreleased changes ) HDFS-3572. Cleanup code which inits SPNEGO in HttpServer (todd) + HDFS-3576. Move the constant NameNode.DEFAULT_PORT to + DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT. (Brandon Li via szetszwo) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java index b31960c974..1d2832a5b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.CorruptFileBlockIterator; import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; @@ -42,7 +43,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.token.SecretManager.InvalidToken; @@ -71,7 +71,8 @@ public class Hdfs extends AbstractFileSystem { * @throws IOException */ Hdfs(final URI theUri, final Configuration conf) throws IOException, URISyntaxException { - super(theUri, HdfsConstants.HDFS_URI_SCHEME, true, NameNode.DEFAULT_PORT); + super(theUri, HdfsConstants.HDFS_URI_SCHEME, true, + DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); if (!theUri.getScheme().equalsIgnoreCase(HdfsConstants.HDFS_URI_SCHEME)) { throw new IllegalArgumentException("Passed URI's scheme is not for Hdfs"); @@ -86,7 +87,7 @@ public class Hdfs extends AbstractFileSystem { @Override public int getUriDefaultPort() { - return NameNode.DEFAULT_PORT; + return DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 146ed8358f..467b345cf4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -87,6 +87,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_HTTP_ADDRESS_KEY = "dfs.namenode.http-address"; public static final String DFS_NAMENODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_NAMENODE_HTTP_PORT_DEFAULT; public static final String DFS_NAMENODE_RPC_ADDRESS_KEY = "dfs.namenode.rpc-address"; + public static final int DFS_NAMENODE_RPC_PORT_DEFAULT = 8020; public static final String DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY = "dfs.namenode.servicerpc-address"; public static final String DFS_NAMENODE_MAX_OBJECTS_KEY = "dfs.namenode.max.objects"; public static final long DFS_NAMENODE_MAX_OBJECTS_DEFAULT = 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 8c0ed10d2b..fb6f9dc10a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -734,7 +734,7 @@ public void setTimes(Path p, long mtime, long atime @Override protected int getDefaultPort() { - return NameNode.DEFAULT_PORT; + return DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java index cc6517daa5..326ba5595d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java @@ -402,7 +402,7 @@ private static Class> getFailoverProxyProviderClass // If we found a proxy provider, then this URI should be a logical NN. // Given that, it shouldn't have a non-default port number. int port = nameNodeUri.getPort(); - if (port > 0 && port != NameNode.DEFAULT_PORT) { + if (port > 0 && port != DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT) { throw new IOException("Port " + port + " specified in URI " + nameNodeUri + " but host '" + host + "' is a logical (HA) namenode" diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java index 293611e377..3822939944 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; @@ -57,7 +57,7 @@ public Token selectToken( Text serviceName = SecurityUtil.buildTokenService(nnUri); final String nnServiceName = conf.get(SERVICE_NAME_KEY + serviceName); - int nnRpcPort = NameNode.DEFAULT_PORT; + int nnRpcPort = DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT; if (nnServiceName != null) { nnRpcPort = NetUtils.createSocketAddr(nnServiceName, nnRpcPort).getPort(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 6416db1d9e..6caf8d3741 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -214,7 +214,6 @@ public long getProtocolVersion(String protocol, } } - public static final int DEFAULT_PORT = 8020; public static final Log LOG = LogFactory.getLog(NameNode.class.getName()); public static final Log stateChangeLog = LogFactory.getLog("org.apache.hadoop.hdfs.StateChange"); public static final HAState ACTIVE_STATE = new ActiveState(); @@ -270,7 +269,7 @@ public static NameNodeMetrics getNameNodeMetrics() { } public static InetSocketAddress getAddress(String address) { - return NetUtils.createSocketAddr(address, DEFAULT_PORT); + return NetUtils.createSocketAddr(address, DFS_NAMENODE_RPC_PORT_DEFAULT); } /** @@ -329,7 +328,8 @@ public static InetSocketAddress getAddress(URI filesystemURI) { public static URI getUri(InetSocketAddress namenode) { int port = namenode.getPort(); - String portString = port == DEFAULT_PORT ? "" : (":"+port); + String portString = (port == DFS_NAMENODE_RPC_PORT_DEFAULT) ? + "" : (":"+port); return URI.create(HdfsConstants.HDFS_URI_SCHEME + "://" + namenode.getHostName()+portString); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java index 38f5123de2..af1a1b70ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java @@ -77,7 +77,7 @@ public NNHAServiceTarget(Configuration conf, "Unable to determine service address for namenode '" + nnId + "'"); } this.addr = NetUtils.createSocketAddr(serviceAddr, - NameNode.DEFAULT_PORT); + DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); this.autoFailoverEnabled = targetConf.getBoolean( DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java index a88e8a74ed..5769f48a57 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java @@ -31,7 +31,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.test.GenericTestUtils; @@ -82,9 +81,9 @@ public void testDfsClientFailover() throws IOException, URISyntaxException { // Check that it functions even if the URL becomes canonicalized // to include a port number. - Path withPort = new Path("hdfs://" + - HATestUtil.getLogicalHostname(cluster) + ":" + - NameNode.DEFAULT_PORT + "/" + TEST_FILE.toUri().getPath()); + Path withPort = new Path("hdfs://" + HATestUtil.getLogicalHostname(cluster) + + ":" + DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT + "/" + + TEST_FILE.toUri().getPath()); FileSystem fs2 = withPort.getFileSystem(fs.getConf()); assertTrue(fs2.exists(withPort)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java index 6a66e947e7..799d28cf6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java @@ -31,9 +31,9 @@ public class TestDefaultNameNodePort extends TestCase { public void testGetAddressFromString() throws Exception { assertEquals(NameNode.getAddress("foo").getPort(), - NameNode.DEFAULT_PORT); + DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); assertEquals(NameNode.getAddress("hdfs://foo/").getPort(), - NameNode.DEFAULT_PORT); + DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); assertEquals(NameNode.getAddress("hdfs://foo:555").getPort(), 555); assertEquals(NameNode.getAddress("foo:555").getPort(), @@ -43,18 +43,20 @@ public void testGetAddressFromString() throws Exception { public void testGetAddressFromConf() throws Exception { Configuration conf = new HdfsConfiguration(); FileSystem.setDefaultUri(conf, "hdfs://foo/"); - assertEquals(NameNode.getAddress(conf).getPort(), NameNode.DEFAULT_PORT); + assertEquals(NameNode.getAddress(conf).getPort(), + DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); FileSystem.setDefaultUri(conf, "hdfs://foo:555/"); assertEquals(NameNode.getAddress(conf).getPort(), 555); FileSystem.setDefaultUri(conf, "foo"); - assertEquals(NameNode.getAddress(conf).getPort(), NameNode.DEFAULT_PORT); + assertEquals(NameNode.getAddress(conf).getPort(), + DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); } public void testGetUri() { assertEquals(NameNode.getUri(new InetSocketAddress("foo", 555)), URI.create("hdfs://foo:555")); assertEquals(NameNode.getUri(new InetSocketAddress("foo", - NameNode.DEFAULT_PORT)), + DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT)), URI.create("hdfs://foo")); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java index 9d13a2b619..c1762d3553 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -32,7 +33,6 @@ import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf; import org.apache.hadoop.hdfs.protocol.ClientProtocol; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.junit.Test; @@ -67,7 +67,7 @@ public void testBalancerWithHANameNodes() throws Exception { assertEquals(capacities.length, racks.length); int numOfDatanodes = capacities.length; NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1"); - nn1Conf.setIpcPort(NameNode.DEFAULT_PORT); + nn1Conf.setIpcPort(DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); Configuration copiedConf = new Configuration(conf); cluster = new MiniDFSCluster.Builder(copiedConf) .nnTopology(MiniDFSNNTopology.simpleHATopology()) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java index f299cb6740..6c978f0b38 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java @@ -39,6 +39,7 @@ import org.apache.commons.logging.Log; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.fs.shell.CommandFormat; @@ -509,10 +510,10 @@ public FileSystem run() throws IOException { { try { - runTestCache(NameNode.DEFAULT_PORT); + runTestCache(DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); } catch(java.net.BindException be) { - LOG.warn("Cannot test NameNode.DEFAULT_PORT (=" - + NameNode.DEFAULT_PORT + ")", be); + LOG.warn("Cannot test NameNode's default RPC port (=" + + DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT + ")", be); } runTestCache(0); @@ -535,11 +536,11 @@ static void runTestCache(int port) throws Exception { } } - if (port == NameNode.DEFAULT_PORT) { + if (port == DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT) { //test explicit default port - URI uri2 = new URI(uri.getScheme(), uri.getUserInfo(), - uri.getHost(), NameNode.DEFAULT_PORT, uri.getPath(), - uri.getQuery(), uri.getFragment()); + URI uri2 = new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), + DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT, uri.getPath(), + uri.getQuery(), uri.getFragment()); LOG.info("uri2=" + uri2); FileSystem fs = FileSystem.get(uri2, conf); checkPath(cluster, fs); From 68833542860e8bdd367a03736de5b3160f6b0b7d Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Thu, 28 Jun 2012 01:25:32 +0000 Subject: [PATCH 91/91] svn merge -c -1354790 for reverting HDFS-3576 since it requires more changes. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1354792 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 --- .../src/main/java/org/apache/hadoop/fs/Hdfs.java | 7 +++---- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 1 - .../apache/hadoop/hdfs/DistributedFileSystem.java | 2 +- .../org/apache/hadoop/hdfs/NameNodeProxies.java | 2 +- .../token/delegation/DelegationTokenSelector.java | 4 ++-- .../hadoop/hdfs/server/namenode/NameNode.java | 6 +++--- .../hadoop/hdfs/tools/NNHAServiceTarget.java | 2 +- .../apache/hadoop/hdfs/TestDFSClientFailover.java | 7 ++++--- .../hadoop/hdfs/TestDefaultNameNodePort.java | 12 +++++------- .../balancer/TestBalancerWithHANameNodes.java | 4 ++-- .../java/org/apache/hadoop/fs/TestFileSystem.java | 15 +++++++-------- 12 files changed, 29 insertions(+), 36 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5cf4a7ddc1..a708333c08 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -250,9 +250,6 @@ Branch-2 ( Unreleased changes ) HDFS-3572. Cleanup code which inits SPNEGO in HttpServer (todd) - HDFS-3576. Move the constant NameNode.DEFAULT_PORT to - DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT. (Brandon Li via szetszwo) - OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java index 1d2832a5b1..b31960c974 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.CorruptFileBlockIterator; import org.apache.hadoop.hdfs.DFSClient; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; @@ -43,6 +42,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.token.SecretManager.InvalidToken; @@ -71,8 +71,7 @@ public class Hdfs extends AbstractFileSystem { * @throws IOException */ Hdfs(final URI theUri, final Configuration conf) throws IOException, URISyntaxException { - super(theUri, HdfsConstants.HDFS_URI_SCHEME, true, - DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); + super(theUri, HdfsConstants.HDFS_URI_SCHEME, true, NameNode.DEFAULT_PORT); if (!theUri.getScheme().equalsIgnoreCase(HdfsConstants.HDFS_URI_SCHEME)) { throw new IllegalArgumentException("Passed URI's scheme is not for Hdfs"); @@ -87,7 +86,7 @@ public class Hdfs extends AbstractFileSystem { @Override public int getUriDefaultPort() { - return DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT; + return NameNode.DEFAULT_PORT; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 467b345cf4..146ed8358f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -87,7 +87,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_HTTP_ADDRESS_KEY = "dfs.namenode.http-address"; public static final String DFS_NAMENODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_NAMENODE_HTTP_PORT_DEFAULT; public static final String DFS_NAMENODE_RPC_ADDRESS_KEY = "dfs.namenode.rpc-address"; - public static final int DFS_NAMENODE_RPC_PORT_DEFAULT = 8020; public static final String DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY = "dfs.namenode.servicerpc-address"; public static final String DFS_NAMENODE_MAX_OBJECTS_KEY = "dfs.namenode.max.objects"; public static final long DFS_NAMENODE_MAX_OBJECTS_DEFAULT = 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index fb6f9dc10a..8c0ed10d2b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -734,7 +734,7 @@ public void setTimes(Path p, long mtime, long atime @Override protected int getDefaultPort() { - return DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT; + return NameNode.DEFAULT_PORT; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java index 326ba5595d..cc6517daa5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java @@ -402,7 +402,7 @@ private static Class> getFailoverProxyProviderClass // If we found a proxy provider, then this URI should be a logical NN. // Given that, it shouldn't have a non-default port number. int port = nameNodeUri.getPort(); - if (port > 0 && port != DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT) { + if (port > 0 && port != NameNode.DEFAULT_PORT) { throw new IOException("Port " + port + " specified in URI " + nameNodeUri + " but host '" + host + "' is a logical (HA) namenode" diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java index 3822939944..293611e377 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; @@ -57,7 +57,7 @@ public Token selectToken( Text serviceName = SecurityUtil.buildTokenService(nnUri); final String nnServiceName = conf.get(SERVICE_NAME_KEY + serviceName); - int nnRpcPort = DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT; + int nnRpcPort = NameNode.DEFAULT_PORT; if (nnServiceName != null) { nnRpcPort = NetUtils.createSocketAddr(nnServiceName, nnRpcPort).getPort(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 6caf8d3741..6416db1d9e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -214,6 +214,7 @@ public long getProtocolVersion(String protocol, } } + public static final int DEFAULT_PORT = 8020; public static final Log LOG = LogFactory.getLog(NameNode.class.getName()); public static final Log stateChangeLog = LogFactory.getLog("org.apache.hadoop.hdfs.StateChange"); public static final HAState ACTIVE_STATE = new ActiveState(); @@ -269,7 +270,7 @@ public static NameNodeMetrics getNameNodeMetrics() { } public static InetSocketAddress getAddress(String address) { - return NetUtils.createSocketAddr(address, DFS_NAMENODE_RPC_PORT_DEFAULT); + return NetUtils.createSocketAddr(address, DEFAULT_PORT); } /** @@ -328,8 +329,7 @@ public static InetSocketAddress getAddress(URI filesystemURI) { public static URI getUri(InetSocketAddress namenode) { int port = namenode.getPort(); - String portString = (port == DFS_NAMENODE_RPC_PORT_DEFAULT) ? - "" : (":"+port); + String portString = port == DEFAULT_PORT ? "" : (":"+port); return URI.create(HdfsConstants.HDFS_URI_SCHEME + "://" + namenode.getHostName()+portString); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java index af1a1b70ed..38f5123de2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java @@ -77,7 +77,7 @@ public NNHAServiceTarget(Configuration conf, "Unable to determine service address for namenode '" + nnId + "'"); } this.addr = NetUtils.createSocketAddr(serviceAddr, - DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); + NameNode.DEFAULT_PORT); this.autoFailoverEnabled = targetConf.getBoolean( DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java index 5769f48a57..a88e8a74ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java @@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.test.GenericTestUtils; @@ -81,9 +82,9 @@ public void testDfsClientFailover() throws IOException, URISyntaxException { // Check that it functions even if the URL becomes canonicalized // to include a port number. - Path withPort = new Path("hdfs://" + HATestUtil.getLogicalHostname(cluster) - + ":" + DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT + "/" - + TEST_FILE.toUri().getPath()); + Path withPort = new Path("hdfs://" + + HATestUtil.getLogicalHostname(cluster) + ":" + + NameNode.DEFAULT_PORT + "/" + TEST_FILE.toUri().getPath()); FileSystem fs2 = withPort.getFileSystem(fs.getConf()); assertTrue(fs2.exists(withPort)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java index 799d28cf6e..6a66e947e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java @@ -31,9 +31,9 @@ public class TestDefaultNameNodePort extends TestCase { public void testGetAddressFromString() throws Exception { assertEquals(NameNode.getAddress("foo").getPort(), - DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); + NameNode.DEFAULT_PORT); assertEquals(NameNode.getAddress("hdfs://foo/").getPort(), - DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); + NameNode.DEFAULT_PORT); assertEquals(NameNode.getAddress("hdfs://foo:555").getPort(), 555); assertEquals(NameNode.getAddress("foo:555").getPort(), @@ -43,20 +43,18 @@ public void testGetAddressFromString() throws Exception { public void testGetAddressFromConf() throws Exception { Configuration conf = new HdfsConfiguration(); FileSystem.setDefaultUri(conf, "hdfs://foo/"); - assertEquals(NameNode.getAddress(conf).getPort(), - DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); + assertEquals(NameNode.getAddress(conf).getPort(), NameNode.DEFAULT_PORT); FileSystem.setDefaultUri(conf, "hdfs://foo:555/"); assertEquals(NameNode.getAddress(conf).getPort(), 555); FileSystem.setDefaultUri(conf, "foo"); - assertEquals(NameNode.getAddress(conf).getPort(), - DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); + assertEquals(NameNode.getAddress(conf).getPort(), NameNode.DEFAULT_PORT); } public void testGetUri() { assertEquals(NameNode.getUri(new InetSocketAddress("foo", 555)), URI.create("hdfs://foo:555")); assertEquals(NameNode.getUri(new InetSocketAddress("foo", - DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT)), + NameNode.DEFAULT_PORT)), URI.create("hdfs://foo")); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java index c1762d3553..9d13a2b619 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java @@ -25,7 +25,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -33,6 +32,7 @@ import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf; import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.junit.Test; @@ -67,7 +67,7 @@ public void testBalancerWithHANameNodes() throws Exception { assertEquals(capacities.length, racks.length); int numOfDatanodes = capacities.length; NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1"); - nn1Conf.setIpcPort(DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); + nn1Conf.setIpcPort(NameNode.DEFAULT_PORT); Configuration copiedConf = new Configuration(conf); cluster = new MiniDFSCluster.Builder(copiedConf) .nnTopology(MiniDFSNNTopology.simpleHATopology()) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java index 6c978f0b38..f299cb6740 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java @@ -39,7 +39,6 @@ import org.apache.commons.logging.Log; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.fs.shell.CommandFormat; @@ -510,10 +509,10 @@ public FileSystem run() throws IOException { { try { - runTestCache(DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); + runTestCache(NameNode.DEFAULT_PORT); } catch(java.net.BindException be) { - LOG.warn("Cannot test NameNode's default RPC port (=" - + DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT + ")", be); + LOG.warn("Cannot test NameNode.DEFAULT_PORT (=" + + NameNode.DEFAULT_PORT + ")", be); } runTestCache(0); @@ -536,11 +535,11 @@ static void runTestCache(int port) throws Exception { } } - if (port == DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT) { + if (port == NameNode.DEFAULT_PORT) { //test explicit default port - URI uri2 = new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), - DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT, uri.getPath(), - uri.getQuery(), uri.getFragment()); + URI uri2 = new URI(uri.getScheme(), uri.getUserInfo(), + uri.getHost(), NameNode.DEFAULT_PORT, uri.getPath(), + uri.getQuery(), uri.getFragment()); LOG.info("uri2=" + uri2); FileSystem fs = FileSystem.get(uri2, conf); checkPath(cluster, fs);