diff --git a/.asf.yaml b/.asf.yaml new file mode 100644 index 0000000000000..91483dfed336c --- /dev/null +++ b/.asf.yaml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +github: + enabled_merge_buttons: + squash: true + merge: false + rebase: false +notifications: + commits: common-commits@hadoop.apache.org + issues: common-issues@hadoop.apache.org + pullrequests: common-issues@hadoop.apache.org + jira_options: link label worklog \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9b2d400d13c9c..ef2d0c1fdb718 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,8 @@ patchprocess/ .history/ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/schema +phantomjsdriver.log #robotframework outputs log.html diff --git a/BUILDING.txt b/BUILDING.txt index c96c851204e75..c34946aa993b7 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -82,7 +82,7 @@ Installing required packages for clean install of Ubuntu 14.04 LTS Desktop: Optional packages: -* Snappy compression +* Snappy compression (only used for hadoop-mapreduce-client-nativetask) $ sudo apt-get install snappy libsnappy-dev * Intel ISA-L library for erasure coding Please refer to https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version @@ -168,7 +168,8 @@ Maven build goals: Snappy is a compression library that can be utilized by the native code. It is currently an optional component, meaning that Hadoop can be built with - or without this dependency. + or without this dependency. Snappy library as optional dependency is only + used for hadoop-mapreduce-client-nativetask. * Use -Drequire.snappy to fail the build if libsnappy.so is not found. If this option is not specified and the snappy library is missing, @@ -436,6 +437,9 @@ Building on CentOS 8 $ sudo dnf group install --with-optional 'Development Tools' $ sudo dnf install java-1.8.0-openjdk-devel maven +* Install python2 for building documentation. + $ sudo dnf install python2 + * Install Protocol Buffers v3.7.1. $ git clone https://github.com/protocolbuffers/protobuf $ cd protobuf @@ -449,8 +453,16 @@ Building on CentOS 8 * Install libraries provided by CentOS 8. $ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel +* Install boost. + $ curl -L -o boost_1_72_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download + $ tar xjf boost_1_72_0.tar.bz2 + $ cd boost_1_72_0 + $ ./bootstrap.sh --prefix=/usr/local + $ ./b2 + $ sudo ./b2 install + * Install optional dependencies (snappy-devel). - $ sudo dnf --enablerepo=PowerTools snappy-devel + $ sudo dnf --enablerepo=PowerTools install snappy-devel * Install optional dependencies (libzstd-devel). $ sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm diff --git a/Jenkinsfile b/Jenkinsfile index 0461c5727aff9..944a35b868b3a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,7 +35,7 @@ pipeline { DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile" YETUS='yetus' // Branch or tag name. Yetus release tags are 'rel/X.Y.Z' - YETUS_VERSION='rel/0.12.0' + YETUS_VERSION='6ab19e71eaf3234863424c6f684b34c1d3dcc0ce' } parameters { @@ -96,8 +96,8 @@ pipeline { YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}") # our project defaults come from a personality file - # which will get loaded automatically by setting the project name YETUS_ARGS+=("--project=hadoop") + YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh") # lots of different output formats YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt") @@ -144,6 +144,7 @@ pipeline { # Dockerfile since we don't want to use the auto-pulled version. YETUS_ARGS+=("--docker") YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") + YETUS_ARGS+=("--mvn-custom-repos") # effectively treat dev-suport as a custom maven module YETUS_ARGS+=("--skip-dirs=dev-support") @@ -159,6 +160,9 @@ pipeline { YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64") YETUS_ARGS+=("--multijdktests=compile") + # custom javadoc goals + YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork") + "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ''' } diff --git a/LICENSE-binary b/LICENSE-binary index dc399df6da7bc..dedd83aeaa6a9 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -405,7 +405,7 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.4.1.min.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js diff --git a/LICENSE.txt b/LICENSE.txt index c8e90f27f293f..3c079898b9071 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -245,7 +245,7 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.4.1.min.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js diff --git a/dev-support/bin/dist-copynativelibs b/dev-support/bin/dist-copynativelibs index ffc82b8fb1b05..7f2b6ad1f5649 100755 --- a/dev-support/bin/dist-copynativelibs +++ b/dev-support/bin/dist-copynativelibs @@ -111,9 +111,6 @@ for i in "$@"; do --openssllibbundle=*) OPENSSLLIBBUNDLE=${i#*=} ;; - --snappybinbundle=*) - SNAPPYBINBUNDLE=${i#*=} - ;; --snappylib=*) SNAPPYLIB=${i#*=} ;; @@ -176,8 +173,6 @@ if [[ -d "${BIN_DIR}" ]] ; then exit 1 fi - bundle_native_bin "${SNAPPYBINBUNDLE}" "${SNAPPYLIBBUNDLE}" "snappy.lib" "snappy" "${SNAPPYLIB}" - bundle_native_bin "${ZSTDBINBUNDLE}" "${ZSTDLIBBUNDLE}" "zstd.lib" "zstd" "${ZSTDLIB}" bundle_native_bin "${OPENSSLBINBUNDLE}" "${OPENSSLLIBBUNDLE}" "openssl.lib" "crypto" "${OPENSSLLIB}" diff --git a/dev-support/bin/hadoop.sh b/dev-support/bin/hadoop.sh new file mode 100755 index 0000000000000..3343014aae8bb --- /dev/null +++ b/dev-support/bin/hadoop.sh @@ -0,0 +1,563 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# SHELLDOC-IGNORE +# +# Override these to match Apache Hadoop's requirements +personality_plugins "all,-ant,-gradle,-scalac,-scaladoc" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + BUILDTOOL=maven + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=trunk + #shellcheck disable=SC2034 + PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute" + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO_DEFAULT="apache/hadoop" + + HADOOP_HOMEBREW_DIR=${HADOOP_HOMEBREW_DIR:-$(brew --prefix 2>/dev/null)} + if [[ -z "${HADOOP_HOMEBREW_DIR}" ]]; then + HADOOP_HOMEBREW_DIR=/usr/local + fi +} + +function personality_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --hadoop-isal-prefix=*) + delete_parameter "${i}" + ISAL_HOME=${i#*=} + ;; + --hadoop-openssl-prefix=*) + delete_parameter "${i}" + OPENSSL_HOME=${i#*=} + ;; + --hadoop-snappy-prefix=*) + delete_parameter "${i}" + SNAPPY_HOME=${i#*=} + ;; + esac + done +} + +## @description Calculate the actual module ordering +## @audience private +## @stability evolving +## @param ordering +function hadoop_order +{ + declare ordering=$1 + declare hadoopm + + if [[ ${ordering} = normal ]]; then + hadoopm="${CHANGED_MODULES[*]}" + elif [[ ${ordering} = union ]]; then + hadoopm="${CHANGED_UNION_MODULES}" + elif [[ ${ordering} = mvnsrc ]]; then + hadoopm="${MAVEN_SRC_MODULES[*]}" + elif [[ ${ordering} = mvnsrctest ]]; then + hadoopm="${MAVEN_SRCTEST_MODULES[*]}" + else + hadoopm="${ordering}" + fi + echo "${hadoopm}" +} + +## @description Determine if it is safe to run parallel tests +## @audience private +## @stability evolving +## @param ordering +function hadoop_test_parallel +{ + if [[ -f "${BASEDIR}/pom.xml" ]]; then + HADOOP_VERSION=$(grep '' "${BASEDIR}/pom.xml" \ + | head -1 \ + | "${SED}" -e 's|^ *||' -e 's|.*$||' \ + | cut -f1 -d- ) + export HADOOP_VERSION + else + return 1 + fi + + hmajor=${HADOOP_VERSION%%\.*} + hmajorminor=${HADOOP_VERSION%\.*} + hminor=${hmajorminor##*\.} + # ... and just for reference + #hmicro=${HADOOP_VERSION##*\.} + + # Apache Hadoop v2.8.0 was the first one to really + # get working parallel unit tests + if [[ ${hmajor} -lt 3 && ${hminor} -lt 8 ]]; then + return 1 + fi + + return 0 +} + +## @description Install extra modules for unit tests +## @audience private +## @stability evolving +## @param ordering +function hadoop_unittest_prereqs +{ + declare input=$1 + declare mods + declare need_common=0 + declare building_common=0 + declare module + declare flags + declare fn + + # prior to running unit tests, hdfs needs libhadoop.so built + # if we're building root, then this extra work is moot + + #shellcheck disable=SC2086 + mods=$(hadoop_order ${input}) + + for module in ${mods}; do + if [[ ${module} = hadoop-hdfs-project* ]]; then + need_common=1 + elif [[ ${module} = hadoop-common-project/hadoop-common + || ${module} = hadoop-common-project ]]; then + building_common=1 + elif [[ ${module} = . ]]; then + return + fi + done + + # Windows builds *ALWAYS* need hadoop-common compiled + case ${OSTYPE} in + Windows_NT|CYGWIN*|MINGW*|MSYS*) + need_common=1 + ;; + esac + + if [[ ${need_common} -eq 1 + && ${building_common} -eq 0 ]]; then + echo "unit test pre-reqs:" + module="hadoop-common-project/hadoop-common" + fn=$(module_file_fragment "${module}") + flags="$(hadoop_native_flags) $(yarn_ui2_flag)" + pushd "${BASEDIR}/${module}" >/dev/null || return 1 + # shellcheck disable=SC2086 + echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags} + popd >/dev/null || return 1 + fi +} + +## @description Calculate the flags/settings for yarn-ui v2 build +## @description based upon the OS +## @audience private +## @stability evolving +function yarn_ui2_flag +{ + + if [[ ${BUILD_NATIVE} != true ]]; then + return + fi + + # Now it only tested on Linux/OSX, don't enable the profile on + # windows until it get verified + case ${OSTYPE} in + Linux) + # shellcheck disable=SC2086 + echo -Pyarn-ui + ;; + Darwin) + echo -Pyarn-ui + ;; + *) + # Do nothing + ;; + esac +} + +## @description Calculate the flags/settings for native code +## @description based upon the OS +## @audience private +## @stability evolving +function hadoop_native_flags +{ + if [[ ${BUILD_NATIVE} != true ]]; then + return + fi + + declare -a args + + # Based upon HADOOP-11937 + # + # Some notes: + # + # - getting fuse to compile on anything but Linux + # is always tricky. + # - Darwin assumes homebrew is in use. + # - HADOOP-12027 required for bzip2 on OS X. + # - bzip2 is broken in lots of places + # (the shared library is considered experimental) + # e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2 + # + + args=("-Drequire.test.libhadoop") + + if [[ -d "${ISAL_HOME}/include" ]]; then + args=("${args[@]}" "-Disal.prefix=${ISAL_HOME}") + fi + + if [[ -d "${OPENSSL_HOME}/include" ]]; then + args=("${args[@]}" "-Dopenssl.prefix=${OPENSSL_HOME}") + elif [[ -d "${HADOOP_HOMEBREW_DIR}/opt/openssl/" ]]; then + args=("${args[@]}" "-Dopenssl.prefix=${HADOOP_HOMEBREW_DIR}/opt/openssl/") + fi + + if [[ -d "${SNAPPY_HOME}/include" ]]; then + args=("${args[@]}" "-Dsnappy.prefix=${SNAPPY_HOME}") + elif [[ -d "${HADOOP_HOMEBREW_DIR}/include/snappy.h" ]]; then + args=("${args[@]}" "-Dsnappy.prefix=${HADOOP_HOMEBREW_DIR}/opt/snappy") + fi + + case ${OSTYPE} in + Linux) + # shellcheck disable=SC2086 + echo \ + -Pnative \ + -Drequire.fuse \ + -Drequire.openssl \ + -Drequire.snappy \ + -Drequire.valgrind \ + -Drequire.zstd \ + "${args[@]}" + ;; + Darwin) + echo \ + "${args[@]}" \ + -Pnative \ + -Drequire.snappy \ + -Drequire.openssl + ;; + Windows_NT|CYGWIN*|MINGW*|MSYS*) + echo \ + "${args[@]}" \ + -Drequire.snappy -Drequire.openssl -Pnative-win + ;; + *) + echo \ + "${args[@]}" + ;; + esac +} + +## @description Queue up modules for this personality +## @audience private +## @stability evolving +## @param repostatus +## @param testtype +function personality_modules +{ + declare repostatus=$1 + declare testtype=$2 + declare extra="" + declare ordering="normal" + declare needflags=false + declare foundbats=false + declare flags + declare fn + declare i + declare hadoopm + + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + case ${testtype} in + asflicense) + # this is very fast and provides the full path if we do it from + # the root of the source + personality_enqueue_module . + return + ;; + checkstyle) + ordering="union" + extra="-DskipTests" + ;; + compile) + ordering="union" + extra="-DskipTests" + needflags=true + + # if something in common changed, we build the whole world + if [[ "${CHANGED_MODULES[*]}" =~ hadoop-common ]]; then + yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . " + ordering="." + fi + ;; + distclean) + ordering="." + extra="-DskipTests" + ;; + javadoc) + if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + + if [[ "${repostatus}" = patch && "${BUILDMODE}" = patch ]]; then + echo "javadoc pre-reqs:" + for i in hadoop-project \ + hadoop-common-project/hadoop-annotations; do + fn=$(module_file_fragment "${i}") + pushd "${BASEDIR}/${i}" >/dev/null || return 1 + echo "cd ${i}" + echo_and_redirect "${PATCH_DIR}/maven-${fn}-install.txt" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" install + popd >/dev/null || return 1 + done + fi + extra="-Pdocs -DskipTests" + ;; + mvneclipse) + if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + ;; + mvninstall) + extra="-DskipTests" + if [[ "${repostatus}" = branch || "${BUILDMODE}" = full ]]; then + ordering=. + fi + ;; + mvnsite) + if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + ;; + unit) + if [[ "${BUILDMODE}" = full ]]; then + ordering=mvnsrc + elif [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + + if [[ ${TEST_PARALLEL} = "true" ]] ; then + if hadoop_test_parallel; then + extra="-Pparallel-tests" + if [[ -n ${TEST_THREADS:-} ]]; then + extra="${extra} -DtestsThreadCount=${TEST_THREADS}" + fi + fi + fi + needflags=true + hadoop_unittest_prereqs "${ordering}" + + if ! verify_needed_test javac; then + yetus_debug "hadoop: javac not requested" + if ! verify_needed_test native; then + yetus_debug "hadoop: native not requested" + yetus_debug "hadoop: adding -DskipTests to unit test" + extra="-DskipTests" + fi + fi + + for i in "${CHANGED_FILES[@]}"; do + if [[ "${i}" =~ \.bats ]]; then + foundbats=true + fi + done + + if ! verify_needed_test shellcheck && [[ ${foundbats} = false ]]; then + yetus_debug "hadoop: NO shell code change detected; disabling shelltest profile" + extra="${extra} -P!shelltest" + else + extra="${extra} -Pshelltest" + fi + ;; + *) + extra="-DskipTests" + ;; + esac + + if [[ ${needflags} = true ]]; then + flags="$(hadoop_native_flags) $(yarn_ui2_flag)" + extra="${extra} ${flags}" + fi + + extra="-Ptest-patch ${extra}" + for module in $(hadoop_order ${ordering}); do + # shellcheck disable=SC2086 + personality_enqueue_module ${module} ${extra} + done +} + +## @description Add tests based upon personality needs +## @audience private +## @stability evolving +## @param filename +function personality_file_tests +{ + declare filename=$1 + + yetus_debug "Using Hadoop-specific personality_file_tests" + + if [[ ${filename} =~ src/main/webapp ]]; then + yetus_debug "tests/webapp: ${filename}" + add_test shadedclient + elif [[ ${filename} =~ \.sh + || ${filename} =~ \.cmd + || ${filename} =~ src/scripts + || ${filename} =~ src/test/scripts + || ${filename} =~ src/main/bin + || ${filename} =~ shellprofile\.d + || ${filename} =~ src/main/conf + ]]; then + yetus_debug "tests/shell: ${filename}" + add_test mvnsite + add_test unit + elif [[ ${filename} =~ \.md$ + || ${filename} =~ \.md\.vm$ + || ${filename} =~ src/site + ]]; then + yetus_debug "tests/site: ${filename}" + add_test mvnsite + elif [[ ${filename} =~ \.c$ + || ${filename} =~ \.cc$ + || ${filename} =~ \.h$ + || ${filename} =~ \.hh$ + || ${filename} =~ \.proto$ + || ${filename} =~ \.cmake$ + || ${filename} =~ CMakeLists.txt + ]]; then + yetus_debug "tests/units: ${filename}" + add_test compile + add_test cc + add_test mvnsite + add_test javac + add_test unit + elif [[ ${filename} =~ build.xml$ + || ${filename} =~ pom.xml$ + || ${filename} =~ \.java$ + || ${filename} =~ src/main + ]]; then + yetus_debug "tests/javadoc+units: ${filename}" + add_test compile + add_test javac + add_test javadoc + add_test mvninstall + add_test mvnsite + add_test unit + add_test shadedclient + fi + + # if we change anything in here, e.g. the test scripts + # then run the client artifact tests + if [[ ${filename} =~ hadoop-client-modules ]]; then + add_test shadedclient + fi + + if [[ ${filename} =~ src/test ]]; then + yetus_debug "tests: src/test" + add_test unit + fi + + if [[ ${filename} =~ \.java$ ]]; then + add_test findbugs + fi +} + +## @description Image to print on success +## @audience private +## @stability evolving +function hadoop_console_success +{ + printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg"; + printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP"; + printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg"; + printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog"; + printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg"; + printf "ICB8bXwgICB8bXwgIAo" +} + +################################################### +# Hadoop project specific check of IT for shaded artifacts + +add_test_type shadedclient + +## @description check for test modules and add test/plugins as needed +## @audience private +## @stability evolving +function shadedclient_initialize +{ + maven_add_install shadedclient +} + +## @description build client facing shaded artifacts and test them +## @audience private +## @stability evolving +## @param repostatus +function shadedclient_rebuild +{ + declare repostatus=$1 + declare logfile="${PATCH_DIR}/${repostatus}-shadedclient.txt" + declare module + declare -a modules=() + + if [[ ${OSTYPE} = Windows_NT || + ${OSTYPE} =~ ^CYGWIN.* || + ${OSTYPE} =~ ^MINGW32.* || + ${OSTYPE} =~ ^MSYS.* ]]; then + echo "hadoop personality: building on windows, skipping check of client artifacts." + return 0 + fi + + yetus_debug "hadoop personality: seeing if we need the test of client artifacts." + for module in hadoop-client-modules/hadoop-client-check-invariants \ + hadoop-client-modules/hadoop-client-check-test-invariants \ + hadoop-client-modules/hadoop-client-integration-tests; do + if [ -d "${module}" ]; then + yetus_debug "hadoop personality: test module '${module}' is present." + modules+=(-pl "${module}") + fi + done + if [ ${#modules[@]} -eq 0 ]; then + echo "hadoop personality: no test modules present, skipping check of client artifacts." + return 0 + fi + + big_console_header "Checking client artifacts on ${repostatus}" + + echo_and_redirect "${logfile}" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \ + "${modules[@]}" \ + -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true + + count=$("${GREP}" -c '\[ERROR\]' "${logfile}") + if [[ ${count} -gt 0 ]]; then + add_vote_table -1 shadedclient "${repostatus} has errors when building and testing our client artifacts." + return 1 + fi + + add_vote_table +1 shadedclient "${repostatus} has no errors when building and testing our client artifacts." + return 0 +} diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index f72fa4659009a..4bce9cf71d729 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -61,7 +61,7 @@ RUN apt-get -q update \ libsnappy-dev \ libssl-dev \ libtool \ - libzstd1-dev \ + libzstd-dev \ locales \ make \ maven \ @@ -129,6 +129,8 @@ ENV PATH "${PATH}:/opt/protobuf/bin" # https://github.com/PyCQA/pylint/issues/2294 #### RUN pip2 install \ + astroid==1.6.6 \ + isort==4.3.21 \ configparser==4.0.2 \ pylint==1.9.2 @@ -138,10 +140,11 @@ RUN pip2 install \ RUN pip2 install python-dateutil==2.7.3 ### -# Install node.js 10.21.0 for web UI framework (4.2.6 ships with Xenial) +# Install node.js 10.x for web UI framework (4.2.6 ships with Xenial) ### +# hadolint ignore=DL3008 RUN curl -L -s -S https://deb.nodesource.com/setup_10.x | bash - \ - && apt-get install -y --no-install-recommends nodejs=10.21.0-1nodesource1 \ + && apt-get install -y --no-install-recommends nodejs \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && npm install -g bower@1.8.8 diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 5fd646fb9c08a..19cfd13b5c763 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -64,7 +64,7 @@ RUN apt-get -q update \ libsnappy-dev \ libssl-dev \ libtool \ - libzstd1-dev \ + libzstd-dev \ locales \ make \ maven \ @@ -132,6 +132,8 @@ ENV PATH "${PATH}:/opt/protobuf/bin" # https://github.com/PyCQA/pylint/issues/2294 #### RUN pip2 install \ + astroid==1.6.6 \ + isort==4.3.21 \ configparser==4.0.2 \ pylint==1.9.2 @@ -141,10 +143,11 @@ RUN pip2 install \ RUN pip2 install python-dateutil==2.7.3 ### -# Install node.js 10.21.0 for web UI framework (4.2.6 ships with Xenial) +# Install node.js 10.x for web UI framework (4.2.6 ships with Xenial) ### +# hadolint ignore=DL3008 RUN curl -L -s -S https://deb.nodesource.com/setup_10.x | bash - \ - && apt-get install -y --no-install-recommends nodejs=10.21.0-1nodesource1 \ + && apt-get install -y --no-install-recommends nodejs \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && npm install -g bower@1.8.8 diff --git a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml index 96e28496d7e98..51f9acc4015ce 100644 --- a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml +++ b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml @@ -121,9 +121,8 @@ - - + + diff --git a/hadoop-client-modules/hadoop-client-api/pom.xml b/hadoop-client-modules/hadoop-client-api/pom.xml index 48bb56efe62d9..8f3de76ca9462 100644 --- a/hadoop-client-modules/hadoop-client-api/pom.xml +++ b/hadoop-client-modules/hadoop-client-api/pom.xml @@ -143,6 +143,8 @@ org/w3c/dom/**/* org/xml/sax/* org/xml/sax/**/* + org/bouncycastle/* + org/bouncycastle/**/* diff --git a/hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index 08f9202972735..d77424e6b7899 100644 --- a/hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -43,6 +43,8 @@ allowed_expr+="|^org/apache/hadoop/" allowed_expr+="|^META-INF/" # * whatever under the "webapps" directory; for things shipped by yarn allowed_expr+="|^webapps/" +# * Resources files used by Hadoop YARN mini cluster +allowed_expr+="|^TERMINAL/" # * Hadoop's default configuration files, which have the form # "_module_-default.xml" allowed_expr+="|^[^-]*-default.xml$" @@ -54,6 +56,8 @@ allowed_expr+="|^org.apache.hadoop.application-classloader.properties$" # * Used by JavaSandboxLinuxContainerRuntime as a default, loaded # from root, so can't relocate. :( allowed_expr+="|^java.policy$" +# * Used by javax.annotation +allowed_expr+="|^jndi.properties$" # * allowing native libraries from rocksdb. Leaving native libraries as it is. allowed_expr+="|^librocksdbjni-linux32.so" allowed_expr+="|^librocksdbjni-linux64.so" diff --git a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml index e58971b25f00c..978918e406773 100644 --- a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml +++ b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml @@ -75,6 +75,27 @@ hadoop-client-minicluster test + + org.bouncycastle + bcprov-jdk15on + test + + + org.bouncycastle + bcpkix-jdk15on + test + + + javax.xml.bind + jaxb-api + test + + + javax.activation + activation + 1.1.1 + test + @@ -159,6 +180,12 @@ test test-jar + + org.apache.hadoop + hadoop-yarn-server-tests + test + test-jar + diff --git a/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java b/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java index 6022fbc688dde..2e304861babbd 100644 --- a/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java +++ b/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.conf.Configuration; @@ -43,6 +44,7 @@ import org.apache.hadoop.hdfs.web.WebHdfsTestUtil; import org.apache.hadoop.hdfs.web.WebHdfsConstants; +import org.apache.hadoop.yarn.server.MiniYARNCluster; /** * Ensure that we can perform operations against the shaded minicluster @@ -54,6 +56,7 @@ public class ITUseMiniCluster { LoggerFactory.getLogger(ITUseMiniCluster.class); private MiniDFSCluster cluster; + private MiniYARNCluster yarnCluster; private static final String TEST_PATH = "/foo/bar/cats/dee"; private static final String FILENAME = "test.file"; @@ -73,6 +76,12 @@ public void clusterUp() throws IOException { .numDataNodes(3) .build(); cluster.waitActive(); + + conf.set("yarn.scheduler.capacity.root.queues", "default"); + conf.setInt("yarn.scheduler.capacity.root.default.capacity", 100); + yarnCluster = new MiniYARNCluster(getClass().getName(), 1, 1, 1, 1); + yarnCluster.init(conf); + yarnCluster.start(); } @After @@ -80,6 +89,7 @@ public void clusterDown() { if (cluster != null) { cluster.close(); } + IOUtils.cleanupWithLogger(LOG, yarnCluster); } @Test diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index f66528dc7f23c..70a627cdc06d2 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -322,6 +322,10 @@ dnsjava dnsjava + + org.apache.hadoop.thirdparty + hadoop-shaded-guava + org.bouncycastle:* @@ -719,13 +729,6 @@ testdata/* - - - org.apache.hadoop:hadoop-yarn-server-nodemanager:* - - TERMINAL/**/* - - @@ -772,6 +775,13 @@ about.html + + + org.eclipse.jetty.websocket:* + + about.html + + org.apache.hadoop:* @@ -869,6 +879,8 @@ org/w3c/dom/**/* org/xml/sax/* org/xml/sax/**/* + org/bouncycastle/* + org/bouncycastle/**/* @@ -965,6 +977,13 @@ **/pom.xml + + javax/annotation/ + ${shaded.dependency.prefix}.javax.websocket. + + **/pom.xml + + jersey/ ${shaded.dependency.prefix}.jersey. diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml index 9a1efff6b1455..ebaafff89bbb3 100644 --- a/hadoop-client-modules/hadoop-client-runtime/pom.xml +++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml @@ -267,6 +267,8 @@ org/w3c/dom/**/* org/xml/sax/* org/xml/sax/**/* + org/bouncycastle/* + org/bouncycastle/**/* diff --git a/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml b/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml index e647e678a07a6..f8c3472640f25 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml +++ b/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml @@ -16,8 +16,8 @@ --> - + - h_LIB + diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileSystem.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileSystem.java index 333b34929ecda..94b10ad44012b 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileSystem.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileSystem.java @@ -32,7 +32,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNOutputStream.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNOutputStream.java index c437dde613d2c..bd449622906f1 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNOutputStream.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNOutputStream.java @@ -32,10 +32,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors; import com.qcloud.cos.model.PartETag; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java index e4c59a5a27611..d2d2f8c9a7cab 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java @@ -24,7 +24,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import com.qcloud.cos.auth.AnonymousCOSCredentials; import com.qcloud.cos.auth.COSCredentials; import com.qcloud.cos.auth.COSCredentialsProvider; diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml index 4ff3bc14927fe..53d8f19de67c1 100644 --- a/hadoop-common-project/hadoop-auth/pom.xml +++ b/hadoop-common-project/hadoop-auth/pom.xml @@ -183,10 +183,15 @@ ${apacheds.version} test + + org.apache.hadoop.thirdparty + hadoop-shaded-guava + compile + com.google.guava guava - compile + test diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java index 64d43307ffc2d..3bfa349880c5a 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java @@ -13,7 +13,7 @@ */ package org.apache.hadoop.security.authentication.client; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.lang.reflect.Constructor; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.security.authentication.server.HttpConstants; diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandlerUtil.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandlerUtil.java index d881c65893058..79739a487b431 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandlerUtil.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandlerUtil.java @@ -20,7 +20,7 @@ import java.util.Locale; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * This is a utility class designed to provide functionality related to diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java index 884398cb799ee..5e4b0e844275a 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java @@ -28,7 +28,7 @@ import java.security.interfaces.RSAPublicKey; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authentication.util.CertificateUtil; import org.slf4j.Logger; diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java index 50eeb2a965e27..703842f3e3915 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java @@ -13,7 +13,7 @@ */ package org.apache.hadoop.security.authentication.server; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; import org.apache.commons.codec.binary.Base64; diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/LdapAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/LdapAuthenticationHandler.java index 8cc8d03447a99..94ed5d44d2a68 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/LdapAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/LdapAuthenticationHandler.java @@ -38,8 +38,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * The {@link LdapAuthenticationHandler} implements the BASIC authentication diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/MultiSchemeAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/MultiSchemeAuthenticationHandler.java index 58a0adb237e70..b2499ff734bbe 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/MultiSchemeAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/MultiSchemeAuthenticationHandler.java @@ -30,8 +30,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Preconditions; -import com.google.common.base.Splitter; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Splitter; /** * The {@link MultiSchemeAuthenticationHandler} supports configuring multiple diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/FileSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/FileSignerSecretProvider.java index e8aa160a20877..c03703732cf08 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/FileSignerSecretProvider.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/FileSignerSecretProvider.java @@ -13,7 +13,7 @@ */ package org.apache.hadoop.security.authentication.util; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java index 67c2c10237d49..a308cef190396 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java @@ -26,7 +26,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.slf4j.Logger; diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java index 9245887832102..a57b744c2be0d 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java @@ -13,7 +13,7 @@ */ package org.apache.hadoop.security.authentication.util; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.security.SecureRandom; import java.util.Random; diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java index e516b5b76a05f..69a09c189be27 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java @@ -18,7 +18,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import javax.servlet.ServletContext; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.slf4j.Logger; diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java index f0c350ed9594b..a1cd6de8e5933 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java @@ -13,7 +13,7 @@ */ package org.apache.hadoop.security.authentication.util; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.nio.ByteBuffer; import java.security.SecureRandom; import java.util.Collections; diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java index 7e5b10e641889..a7747398eec46 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java @@ -17,7 +17,7 @@ import java.util.Properties; import javax.servlet.ServletContext; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProviderCreator.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProviderCreator.java index e567e7bfbafb3..cb59c2099fc2c 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProviderCreator.java +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProviderCreator.java @@ -13,7 +13,7 @@ */ package org.apache.hadoop.security.authentication.util; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceStability; /** diff --git a/hadoop-common-project/hadoop-common/HadoopCommon.cmake b/hadoop-common-project/hadoop-common/HadoopCommon.cmake index 4de70ac3f78e2..7628ecf628de6 100644 --- a/hadoop-common-project/hadoop-common/HadoopCommon.cmake +++ b/hadoop-common-project/hadoop-common/HadoopCommon.cmake @@ -193,7 +193,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") elseif(CMAKE_SYSTEM_NAME STREQUAL "SunOS") # Solaris flags. 64-bit compilation is mandatory, and is checked earlier. hadoop_add_compiler_flags("-m64 -D_POSIX_C_SOURCE=200112L -D__EXTENSIONS__ -D_POSIX_PTHREAD_SEMANTICS") - set(CMAKE_CXX_FLAGS "-std=gnu++98 ${CMAKE_CXX_FLAGS}") + set(CMAKE_CXX_STANDARD 98) hadoop_add_linker_flags("-m64") # CMAKE_SYSTEM_PROCESSOR is set to the output of 'uname -p', which on Solaris is @@ -212,4 +212,4 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "SunOS") endif() # Set GNU99 as the C standard to use -set(CMAKE_C_FLAGS "-std=gnu99 ${CMAKE_C_FLAGS}") \ No newline at end of file +set(CMAKE_C_STANDARD 99) \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch b/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch index 8f87d4092bcd3..2bd7b63f0178f 100644 --- a/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch +++ b/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch @@ -1,11 +1,12 @@ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java -index a277abd..ed7c709 100644 +index a277abd6e13..1d131d5db6e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java -@@ -43,18 +43,6 @@ +@@ -42,18 +42,6 @@ + @InterfaceAudience.Private public abstract MetricsSystem init(String prefix); - /** +- /** - * Register a metrics source - * @param the actual type of the source object - * @param source object to register @@ -17,15 +18,11 @@ index a277abd..ed7c709 100644 - */ - public abstract T register(String name, String desc, T source); - -- /** + /** * Unregister a metrics source * @param name of the source. This is the name you use to call register() - */ -@@ -77,18 +65,19 @@ - */ - @InterfaceAudience.Private +@@ -79,16 +67,16 @@ public abstract MetricsSource getSource(String name); -+ /** - * Register a metrics sink @@ -50,10 +47,10 @@ index a277abd..ed7c709 100644 /** * Register a callback interface for JMX events diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java -index 6986edb..eeea81f 100644 +index a6edf08e5a7..5b87be1ec67 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java -@@ -270,27 +270,6 @@ void registerSource(String name, String desc, MetricsSource source) { +@@ -269,31 +269,6 @@ void registerSource(String name, String desc, MetricsSource source) { LOG.debug("Registered source "+ name); } @@ -61,7 +58,11 @@ index 6986edb..eeea81f 100644 - T register(final String name, final String description, final T sink) { - LOG.debug(name +", "+ description); - if (allSinks.containsKey(name)) { -- LOG.warn("Sink "+ name +" already exists!"); +- if(sinks.get(name) == null) { +- registerSink(name, description, sink); +- } else { +- LOG.warn("Sink "+ name +" already exists!"); +- } - return sink; - } - allSinks.put(name, sink); @@ -82,10 +83,10 @@ index 6986edb..eeea81f 100644 checkNotNull(config, "config"); MetricsConfig conf = sinkConfigs.get(name); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java -index c19d238..f8412f1 100644 +index 7bc772f062a..f012c877d18 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java -@@ -130,8 +130,8 @@ public void testTagsForPrefix() throws Exception { +@@ -139,8 +139,8 @@ public void testTagsForPrefix() throws Exception { GangliaMetricsTestHelper.setDatagramSocket(gsink31, mockds31); // register the sinks diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.1.2.xml b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.1.2.xml index e750a5c431b67..05ea96a63921c 100644 --- a/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.1.2.xml +++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.1.2.xml @@ -24379,7 +24379,7 @@ to databases, and to files.]]> static="false" final="false" visibility="public" deprecated="not deprecated"> - diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 9bb70ac76a06a..cc786e8750767 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -48,6 +48,11 @@ hadoop-annotations compile + + org.apache.hadoop.thirdparty + hadoop-shaded-guava + + com.google.guava guava @@ -322,7 +327,6 @@ org.bouncycastle bcprov-jdk15on - test org.apache.kerby @@ -362,6 +366,11 @@ wildfly-openssl-java provided + + org.xerial.snappy + snappy-java + compile + @@ -641,10 +650,6 @@ false - - - - false @@ -698,11 +703,7 @@ ${project.build.directory}/native/javah ${sun.arch.data.model} ${require.bzip2} - ${require.snappy} ${require.zstd} - ${snappy.prefix} - ${snappy.lib} - ${snappy.include} ${zstd.prefix} ${zstd.lib} ${zstd.include} @@ -757,14 +758,9 @@ - - - false - false - true @@ -864,10 +860,6 @@ /nologo /p:Configuration=Release /p:OutDir=${project.build.directory}/bin/ - /p:CustomSnappyPrefix=${snappy.prefix} - /p:CustomSnappyLib=${snappy.lib} - /p:CustomSnappyInclude=${snappy.include} - /p:RequireSnappy=${require.snappy} /p:CustomZstdPrefix=${zstd.prefix} /p:CustomZstdLib=${zstd.lib} /p:CustomZstdInclude=${zstd.include} diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt index 10591f6ce2aa8..71c950b7f12ca 100644 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -67,33 +67,6 @@ else() endif() set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES}) -# Require snappy. -set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) -hadoop_set_find_shared_library_version("1") -find_library(SNAPPY_LIBRARY - NAMES snappy - PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/lib - ${CUSTOM_SNAPPY_PREFIX}/lib64 ${CUSTOM_SNAPPY_LIB}) -set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES}) -find_path(SNAPPY_INCLUDE_DIR - NAMES snappy.h - PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/include - ${CUSTOM_SNAPPY_INCLUDE}) -if(SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR) - get_filename_component(HADOOP_SNAPPY_LIBRARY ${SNAPPY_LIBRARY} NAME) - set(SNAPPY_SOURCE_FILES - "${SRC}/io/compress/snappy/SnappyCompressor.c" - "${SRC}/io/compress/snappy/SnappyDecompressor.c") - set(REQUIRE_SNAPPY ${REQUIRE_SNAPPY}) # Stop warning about unused variable. - message(STATUS "Found Snappy: ${SNAPPY_LIBRARY}") -else() - set(SNAPPY_INCLUDE_DIR "") - set(SNAPPY_SOURCE_FILES "") - if(REQUIRE_SNAPPY) - message(FATAL_ERROR "Required snappy library could not be found. SNAPPY_LIBRARY=${SNAPPY_LIBRARY}, SNAPPY_INCLUDE_DIR=${SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_INCLUDE_DIR=${CUSTOM_SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_PREFIX=${CUSTOM_SNAPPY_PREFIX}, CUSTOM_SNAPPY_INCLUDE=${CUSTOM_SNAPPY_INCLUDE}") - endif() -endif() - # Require zstandard SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) hadoop_set_find_shared_library_version("1") @@ -253,7 +226,6 @@ include_directories( ${JNI_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS} ${BZIP2_INCLUDE_DIR} - ${SNAPPY_INCLUDE_DIR} ${ISAL_INCLUDE_DIR} ${ZSTD_INCLUDE_DIR} ${OPENSSL_INCLUDE_DIR} @@ -269,7 +241,6 @@ hadoop_add_dual_library(hadoop ${SRC}/io/compress/lz4/lz4.c ${SRC}/io/compress/lz4/lz4hc.c ${ISAL_SOURCE_FILES} - ${SNAPPY_SOURCE_FILES} ${ZSTD_SOURCE_FILES} ${OPENSSL_SOURCE_FILES} ${SRC}/io/compress/zlib/ZlibCompressor.c diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java index cce744e5076f5..836f3819f8baf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java @@ -30,7 +30,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.http.HttpServer2; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * A servlet to print out the running configuration data. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index b93581e8a7b98..e0e7ac3960451 100755 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -24,7 +24,7 @@ import com.ctc.wstx.stax.WstxInputFactory; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.io.BufferedInputStream; import java.io.DataInput; @@ -40,6 +40,8 @@ import java.lang.ref.WeakReference; import java.net.InetSocketAddress; import java.net.JarURLConnection; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; import java.nio.file.Files; @@ -81,7 +83,7 @@ import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import org.apache.commons.collections.map.UnmodifiableMap; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -105,8 +107,8 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; @@ -1025,11 +1027,11 @@ public synchronized void reloadConfiguration() { properties = null; // trigger reload finalParameters.clear(); // clear site-limits } - + private synchronized void addResourceObject(Resource resource) { resources.add(resource); // add to resources restrictSystemProps |= resource.isParserRestricted(); - reloadConfiguration(); + loadProps(properties, resources.size() - 1, false); } private static final int MAX_SUBST = 20; @@ -2874,12 +2876,27 @@ public Set getFinalParameters() { protected synchronized Properties getProps() { if (properties == null) { properties = new Properties(); - Map backup = updatingResource != null ? - new ConcurrentHashMap(updatingResource) : null; - loadResources(properties, resources, quietmode); + loadProps(properties, 0, true); + } + return properties; + } + /** + * Loads the resource at a given index into the properties. + * @param props the object containing the loaded properties. + * @param startIdx the index where the new resource has been added. + * @param fullReload flag whether we do complete reload of the conf instead + * of just loading the new resource. + */ + private synchronized void loadProps(final Properties props, + final int startIdx, final boolean fullReload) { + if (props != null) { + Map backup = + updatingResource != null + ? new ConcurrentHashMap<>(updatingResource) : null; + loadResources(props, resources, startIdx, fullReload, quietmode); if (overlay != null) { - properties.putAll(overlay); + props.putAll(overlay); if (backup != null) { for (Map.Entry item : overlay.entrySet()) { String key = (String) item.getKey(); @@ -2891,7 +2908,6 @@ protected synchronized Properties getProps() { } } } - return properties; } /** @@ -2993,14 +3009,16 @@ private XMLStreamReader parse(InputStream is, String systemIdStr, private void loadResources(Properties properties, ArrayList resources, + int startIdx, + boolean fullReload, boolean quiet) { - if(loadDefaults) { + if(loadDefaults && fullReload) { for (String resource : defaultResources) { loadResource(properties, new Resource(resource, false), quiet); } } - for (int i = 0; i < resources.size(); i++) { + for (int i = startIdx; i < resources.size(); i++) { Resource ret = loadResource(properties, resources.get(i), quiet); if (ret != null) { resources.set(i, ret); @@ -3247,7 +3265,15 @@ private void handleInclude() throws XMLStreamException, IOException { File href = new File(confInclude); if (!href.isAbsolute()) { // Included resources are relative to the current resource - File baseFile = new File(name).getParentFile(); + File baseFile; + + try { + baseFile = new File(new URI(name)); + } catch (IllegalArgumentException | URISyntaxException e) { + baseFile = new File(name); + } + + baseFile = baseFile.getParentFile(); href = new File(baseFile, href.getPath()); } if (!href.exists()) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java index 8cacbdcdac039..0352025ada154 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java @@ -18,9 +18,9 @@ package org.apache.hadoop.conf; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.util.Time; import org.apache.hadoop.conf.ReconfigurationUtil.PropertyChange; import org.slf4j.Logger; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AesCtrCryptoCodec.java deleted file mode 100644 index 3e52560259638..0000000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AesCtrCryptoCodec.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.crypto; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -import com.google.common.base.Preconditions; - -import java.io.IOException; - -@InterfaceAudience.Private -@InterfaceStability.Evolving -public abstract class AesCtrCryptoCodec extends CryptoCodec { - - protected static final CipherSuite SUITE = CipherSuite.AES_CTR_NOPADDING; - - /** - * For AES, the algorithm block is fixed size of 128 bits. - * @see http://en.wikipedia.org/wiki/Advanced_Encryption_Standard - */ - private static final int AES_BLOCK_SIZE = SUITE.getAlgorithmBlockSize(); - - @Override - public CipherSuite getCipherSuite() { - return SUITE; - } - - /** - * The IV is produced by adding the initial IV to the counter. IV length - * should be the same as {@link #AES_BLOCK_SIZE} - */ - @Override - public void calculateIV(byte[] initIV, long counter, byte[] IV) { - Preconditions.checkArgument(initIV.length == AES_BLOCK_SIZE); - Preconditions.checkArgument(IV.length == AES_BLOCK_SIZE); - - int i = IV.length; // IV length - int j = 0; // counter bytes index - int sum = 0; - while (i-- > 0) { - // (sum >>> Byte.SIZE) is the carry for addition - sum = (initIV[i] & 0xff) + (sum >>> Byte.SIZE); - if (j++ < 8) { // Big-endian, and long is 8 bytes length - sum += (byte) counter & 0xff; - counter >>>= 8; - } - IV[i] = (byte) sum; - } - } - - @Override - public void close() throws IOException { - } -} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java index 8221ba2bd7e26..14a82a2a5d382 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java @@ -28,7 +28,8 @@ @InterfaceAudience.Private public enum CipherSuite { UNKNOWN("Unknown", 0), - AES_CTR_NOPADDING("AES/CTR/NoPadding", 16); + AES_CTR_NOPADDING("AES/CTR/NoPadding", 16), + SM4_CTR_NOPADDING("SM4/CTR/NoPadding", 16); private final String name; private final int algoBlockSize; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java index bcf4a65ec24d4..111e91b5c969e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java @@ -31,8 +31,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Splitter; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY; @@ -101,7 +101,7 @@ public static CryptoCodec getInstance(Configuration conf) { HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT); return getInstance(conf, CipherSuite.convert(name)); } - + private static List> getCodecClasses( Configuration conf, CipherSuite cipherSuite) { List> result = Lists.newArrayList(); @@ -112,6 +112,10 @@ private static List> getCodecClasses( .HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY)) { codecString = conf.get(configName, CommonConfigurationKeysPublic .HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_DEFAULT); + } else if (configName.equals(CommonConfigurationKeysPublic + .HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY)){ + codecString = conf.get(configName, CommonConfigurationKeysPublic + .HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_DEFAULT); } else { codecString = conf.get(configName); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java index 9e601e26cf944..30817a2a62529 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java @@ -30,7 +30,7 @@ import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.ByteBufferPositionedReadable; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java index 8d11043937612..aeb6e4d0ed2ef 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java @@ -29,7 +29,7 @@ import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.fs.Syncable; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * CryptoOutputStream encrypts data. It is not thread-safe. AES CTR mode is diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoStreamUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoStreamUtils.java index b55f84226d3cd..7d8139f8eadd5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoStreamUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoStreamUtils.java @@ -29,7 +29,7 @@ import org.apache.hadoop.fs.Seekable; import org.apache.hadoop.util.CleanerUtil; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,10 +58,12 @@ public static int getBufferSize(Configuration conf) { HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_DEFAULT); } - /** AES/CTR/NoPadding is required */ + /** AES/CTR/NoPadding or SM4/CTR/NoPadding is required. */ public static void checkCodec(CryptoCodec codec) { - if (codec.getCipherSuite() != CipherSuite.AES_CTR_NOPADDING) { - throw new UnsupportedCodecException("AES/CTR/NoPadding is required"); + if (codec.getCipherSuite() != CipherSuite.AES_CTR_NOPADDING && + codec.getCipherSuite() != CipherSuite.SM4_CTR_NOPADDING) { + throw new UnsupportedCodecException( + "AES/CTR/NoPadding or SM4/CTR/NoPadding is required"); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceAesCtrCryptoCodec.java index de0e5dd6268f2..47383abea7579 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceAesCtrCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceAesCtrCryptoCodec.java @@ -17,149 +17,49 @@ */ package org.apache.hadoop.crypto; -import java.io.IOException; -import java.nio.ByteBuffer; import java.security.GeneralSecurityException; -import java.security.SecureRandom; - import javax.crypto.Cipher; -import javax.crypto.spec.IvParameterSpec; -import javax.crypto.spec.SecretKeySpec; - import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; - -import com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT; - /** * Implement the AES-CTR crypto codec using JCE provider. */ @InterfaceAudience.Private -public class JceAesCtrCryptoCodec extends AesCtrCryptoCodec { +public class JceAesCtrCryptoCodec extends JceCtrCryptoCodec { + private static final Logger LOG = LoggerFactory.getLogger(JceAesCtrCryptoCodec.class.getName()); - - private Configuration conf; - private String provider; - private SecureRandom random; public JceAesCtrCryptoCodec() { } - + + @Override + public Logger getLogger() { + return LOG; + } + @Override - public Configuration getConf() { - return conf; + public CipherSuite getCipherSuite() { + return CipherSuite.AES_CTR_NOPADDING; } - + @Override - public void setConf(Configuration conf) { - this.conf = conf; - provider = conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY); - final String secureRandomAlg = conf.get( - HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY, - HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT); - try { - random = (provider != null) ? - SecureRandom.getInstance(secureRandomAlg, provider) : - SecureRandom.getInstance(secureRandomAlg); - } catch (GeneralSecurityException e) { - LOG.warn(e.getMessage()); - random = new SecureRandom(); - } + public void calculateIV(byte[] initIV, long counter, byte[] iv) { + super.calculateIV(initIV, counter, iv, + getCipherSuite().getAlgorithmBlockSize()); } @Override public Encryptor createEncryptor() throws GeneralSecurityException { - return new JceAesCtrCipher(Cipher.ENCRYPT_MODE, provider); + return new JceCtrCipher(Cipher.ENCRYPT_MODE, getProvider(), + getCipherSuite(), "AES"); } @Override public Decryptor createDecryptor() throws GeneralSecurityException { - return new JceAesCtrCipher(Cipher.DECRYPT_MODE, provider); - } - - @Override - public void generateSecureRandom(byte[] bytes) { - random.nextBytes(bytes); - } - - private static class JceAesCtrCipher implements Encryptor, Decryptor { - private final Cipher cipher; - private final int mode; - private boolean contextReset = false; - - public JceAesCtrCipher(int mode, String provider) - throws GeneralSecurityException { - this.mode = mode; - if (provider == null || provider.isEmpty()) { - cipher = Cipher.getInstance(SUITE.getName()); - } else { - cipher = Cipher.getInstance(SUITE.getName(), provider); - } - } - - @Override - public void init(byte[] key, byte[] iv) throws IOException { - Preconditions.checkNotNull(key); - Preconditions.checkNotNull(iv); - contextReset = false; - try { - cipher.init(mode, new SecretKeySpec(key, "AES"), - new IvParameterSpec(iv)); - } catch (Exception e) { - throw new IOException(e); - } - } - - /** - * AES-CTR will consume all of the input data. It requires enough space in - * the destination buffer to encrypt entire input buffer. - */ - @Override - public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) - throws IOException { - process(inBuffer, outBuffer); - } - - /** - * AES-CTR will consume all of the input data. It requires enough space in - * the destination buffer to decrypt entire input buffer. - */ - @Override - public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) - throws IOException { - process(inBuffer, outBuffer); - } - - private void process(ByteBuffer inBuffer, ByteBuffer outBuffer) - throws IOException { - try { - int inputSize = inBuffer.remaining(); - // Cipher#update will maintain crypto context. - int n = cipher.update(inBuffer, outBuffer); - if (n < inputSize) { - /** - * Typically code will not get here. Cipher#update will consume all - * input data and put result in outBuffer. - * Cipher#doFinal will reset the crypto context. - */ - contextReset = true; - cipher.doFinal(inBuffer, outBuffer); - } - } catch (Exception e) { - throw new IOException(e); - } - } - - @Override - public boolean isContextReset() { - return contextReset; - } + return new JceCtrCipher(Cipher.DECRYPT_MODE, getProvider(), + getCipherSuite(), "AES"); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java new file mode 100644 index 0000000000000..3c4b0f13f9cf2 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto; + +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.GeneralSecurityException; +import java.security.SecureRandom; +import java.security.Security; +import javax.crypto.Cipher; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import org.slf4j.Logger; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public abstract class JceCtrCryptoCodec extends CryptoCodec{ + private Configuration conf; + private String provider; + private SecureRandom random; + + public String getProvider() { + return provider; + } + + public void setProvider(String provider) { + this.provider = provider; + } + + public void calculateIV(byte[] initIV, long counter, + byte[] iv, int blockSize) { + Preconditions.checkArgument(initIV.length == blockSize); + Preconditions.checkArgument(iv.length == blockSize); + + int i = iv.length; // IV length + int j = 0; // counter bytes index + int sum = 0; + while(i-- > 0) { + // (sum >>> Byte.SIZE) is the carry for condition + sum = (initIV[i] & 0xff) + (sum >>> Byte.SIZE); + if (j++ < 8) { // Big-endian, and long is 8 bytes length + sum += (byte) counter & 0xff; + counter >>>= 8; + } + iv[i] = (byte) sum; + } + } + + public void close() throws IOException { + } + + protected abstract Logger getLogger(); + + public Configuration getConf() { + return conf; + } + + public void setConf(Configuration conf) { + this.conf = conf; + setProvider(conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY)); + if (BouncyCastleProvider.PROVIDER_NAME.equals(provider)) { + Security.addProvider(new BouncyCastleProvider()); + } + final String secureRandomAlg = + conf.get( + HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY, + HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT); + + try { + random = (provider != null) + ? SecureRandom.getInstance(secureRandomAlg, provider) + : SecureRandom.getInstance(secureRandomAlg); + } catch(GeneralSecurityException e) { + getLogger().warn(e.getMessage()); + random = new SecureRandom(); + } + } + + @Override + public void generateSecureRandom(byte[] bytes) { + random.nextBytes(bytes); + } + + protected static class JceCtrCipher implements Encryptor, Decryptor { + private final Cipher cipher; + private final int mode; + private String name; + private boolean contextReset = false; + + public JceCtrCipher(int mode, String provider, + CipherSuite suite, String name) + throws GeneralSecurityException { + + this.mode = mode; + this.name = name; + if(provider == null || provider.isEmpty()) { + cipher = Cipher.getInstance(suite.getName()); + } else { + cipher = Cipher.getInstance(suite.getName(), provider); + } + } + + public void init(byte[] key, byte[] iv) throws IOException { + Preconditions.checkNotNull(key); + Preconditions.checkNotNull(iv); + contextReset = false; + try { + cipher.init(mode, new SecretKeySpec(key, name), + new IvParameterSpec(iv)); + } catch (Exception e) { + throw new IOException(e); + } + } + + public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) + throws IOException { + process(inBuffer, outBuffer); + } + + public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) + throws IOException { + process(inBuffer, outBuffer); + } + + public void process(ByteBuffer inBuffer, ByteBuffer outBuffer) + throws IOException { + try { + int inputSize = inBuffer.remaining(); + // Cipher#update will maintain crypto context. + int n = cipher.update(inBuffer, outBuffer); + if (n < inputSize) { + /** + * Typically code will not get here. Cipher#update will consume all + * input data and put result in outBuffer. + * Cipher#doFinal will reset the crypto context. + */ + contextReset = true; + cipher.doFinal(inBuffer, outBuffer); + } + } catch (Exception e) { + throw new IOException(e); + } + } + + public boolean isContextReset() { + return contextReset; + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceSm4CtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceSm4CtrCryptoCodec.java new file mode 100644 index 0000000000000..dc6680b3f7e61 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceSm4CtrCryptoCodec.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import javax.crypto.Cipher; +import java.security.GeneralSecurityException; + +/** + * Implement the SM4-CTR crypto codec using JCE provider. + */ +@InterfaceAudience.Private +public class JceSm4CtrCryptoCodec extends JceCtrCryptoCodec { + + private static final Logger LOG = + LoggerFactory.getLogger(JceSm4CtrCryptoCodec.class.getName()); + + public JceSm4CtrCryptoCodec() { + } + + @Override + public Logger getLogger() { + return LOG; + } + + @Override + public CipherSuite getCipherSuite() { + return CipherSuite.SM4_CTR_NOPADDING; + } + + @Override + public void calculateIV(byte[] initIV, long counter, byte[] iv) { + super.calculateIV(initIV, counter, iv, + getCipherSuite().getAlgorithmBlockSize()); + } + + @Override + public Encryptor createEncryptor() throws GeneralSecurityException { + return new JceCtrCipher(Cipher.ENCRYPT_MODE, getProvider(), + getCipherSuite(), "SM4"); + } + + @Override + public Decryptor createDecryptor() throws GeneralSecurityException { + return new JceCtrCipher(Cipher.DECRYPT_MODE, getProvider(), + getCipherSuite(), "SM4"); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java index a127925a7a538..9839898752194 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java @@ -17,35 +17,21 @@ */ package org.apache.hadoop.crypto; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY; - -import java.io.Closeable; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.security.GeneralSecurityException; -import java.security.SecureRandom; -import java.util.Random; - import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.crypto.random.OpensslSecureRandom; -import org.apache.hadoop.util.ReflectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.security.GeneralSecurityException; + /** * Implement the AES-CTR crypto codec using JNI into OpenSSL. */ @InterfaceAudience.Private -public class OpensslAesCtrCryptoCodec extends AesCtrCryptoCodec { +public class OpensslAesCtrCryptoCodec extends OpensslCtrCryptoCodec { + private static final Logger LOG = - LoggerFactory.getLogger(OpensslAesCtrCryptoCodec.class.getName()); + LoggerFactory.getLogger(OpensslAesCtrCryptoCodec.class.getName()); - private Configuration conf; - private Random random; - public OpensslAesCtrCryptoCodec() { String loadingFailureReason = OpensslCipher.getLoadingFailureReason(); if (loadingFailureReason != null) { @@ -54,114 +40,30 @@ public OpensslAesCtrCryptoCodec() { } @Override - public void setConf(Configuration conf) { - this.conf = conf; - final Class klass = conf.getClass( - HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY, OpensslSecureRandom.class, - Random.class); - try { - random = ReflectionUtils.newInstance(klass, conf); - if (LOG.isDebugEnabled()) { - LOG.debug("Using " + klass.getName() + " as random number generator."); - } - } catch (Exception e) { - LOG.info("Unable to use " + klass.getName() + ". Falling back to " + - "Java SecureRandom.", e); - this.random = new SecureRandom(); - } + public Logger getLogger() { + return LOG; } @Override - public Configuration getConf() { - return conf; + public CipherSuite getCipherSuite() { + return CipherSuite.AES_CTR_NOPADDING; } @Override - public Encryptor createEncryptor() throws GeneralSecurityException { - return new OpensslAesCtrCipher(OpensslCipher.ENCRYPT_MODE); + public void calculateIV(byte[] initIV, long counter, byte[] iv) { + super.calculateIV(initIV, counter, iv, + getCipherSuite().getAlgorithmBlockSize()); } @Override - public Decryptor createDecryptor() throws GeneralSecurityException { - return new OpensslAesCtrCipher(OpensslCipher.DECRYPT_MODE); - } - - @Override - public void generateSecureRandom(byte[] bytes) { - random.nextBytes(bytes); + public Encryptor createEncryptor() throws GeneralSecurityException { + return new OpensslCtrCipher(OpensslCipher.ENCRYPT_MODE, + getCipherSuite()); } @Override - public void close() throws IOException { - try { - Closeable r = (Closeable) this.random; - r.close(); - } catch (ClassCastException e) { - } - super.close(); - } - - private static class OpensslAesCtrCipher implements Encryptor, Decryptor { - private final OpensslCipher cipher; - private final int mode; - private boolean contextReset = false; - - public OpensslAesCtrCipher(int mode) throws GeneralSecurityException { - this.mode = mode; - cipher = OpensslCipher.getInstance(SUITE.getName()); - } - - @Override - public void init(byte[] key, byte[] iv) throws IOException { - Preconditions.checkNotNull(key); - Preconditions.checkNotNull(iv); - contextReset = false; - cipher.init(mode, key, iv); - } - - /** - * AES-CTR will consume all of the input data. It requires enough space in - * the destination buffer to encrypt entire input buffer. - */ - @Override - public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) - throws IOException { - process(inBuffer, outBuffer); - } - - /** - * AES-CTR will consume all of the input data. It requires enough space in - * the destination buffer to decrypt entire input buffer. - */ - @Override - public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) - throws IOException { - process(inBuffer, outBuffer); - } - - private void process(ByteBuffer inBuffer, ByteBuffer outBuffer) - throws IOException { - try { - int inputSize = inBuffer.remaining(); - // OpensslCipher#update will maintain crypto context. - int n = cipher.update(inBuffer, outBuffer); - if (n < inputSize) { - /** - * Typically code will not get here. OpensslCipher#update will - * consume all input data and put result in outBuffer. - * OpensslCipher#doFinal will reset the crypto context. - */ - contextReset = true; - cipher.doFinal(outBuffer); - } - } catch (Exception e) { - throw new IOException(e); - } - } - - @Override - public boolean isContextReset() { - return contextReset; - } + public Decryptor createDecryptor() throws GeneralSecurityException { + return new OpensslCtrCipher(OpensslCipher.DECRYPT_MODE, + getCipherSuite()); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java index 0a2ba52e555e5..7cf2afdc31624 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java @@ -29,7 +29,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.util.NativeCodeLoader; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.util.PerformanceAdvisory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,10 +45,11 @@ public final class OpensslCipher { LoggerFactory.getLogger(OpensslCipher.class.getName()); public static final int ENCRYPT_MODE = 1; public static final int DECRYPT_MODE = 0; - - /** Currently only support AES/CTR/NoPadding. */ + + /** Currently only support AES/CTR/NoPadding and SM4/CTR/NoPadding. */ private enum AlgMode { - AES_CTR; + AES_CTR, + SM4_CTR; static int get(String algorithm, String mode) throws NoSuchAlgorithmException { @@ -76,6 +77,7 @@ static int get(String padding) throws NoSuchPaddingException { private long context = 0; private final int alg; private final int padding; + private long engine; private static final String loadingFailureReason; @@ -100,10 +102,16 @@ public static String getLoadingFailureReason() { return loadingFailureReason; } - private OpensslCipher(long context, int alg, int padding) { + private OpensslCipher(long context, int alg, int padding, long engine) { this.context = context; this.alg = alg; this.padding = padding; + this.engine = engine; + } + + public static OpensslCipher getInstance(String transformation) + throws NoSuchAlgorithmException, NoSuchPaddingException { + return getInstance(transformation, null); } /** @@ -112,6 +120,8 @@ private OpensslCipher(long context, int alg, int padding) { * * @param transformation the name of the transformation, e.g., * AES/CTR/NoPadding. + * @param engineId the openssl engine to use.if not set, + * defalut engine will be used. * @return OpensslCipher an OpensslCipher object * @throws NoSuchAlgorithmException if transformation is null, * empty, in an invalid format, or if Openssl doesn't implement the @@ -119,13 +129,15 @@ private OpensslCipher(long context, int alg, int padding) { * @throws NoSuchPaddingException if transformation contains * a padding scheme that is not available. */ - public static final OpensslCipher getInstance(String transformation) + public static OpensslCipher getInstance( + String transformation, String engineId) throws NoSuchAlgorithmException, NoSuchPaddingException { Transform transform = tokenizeTransformation(transformation); int algMode = AlgMode.get(transform.alg, transform.mode); int padding = Padding.get(transform.padding); long context = initContext(algMode, padding); - return new OpensslCipher(context, algMode, padding); + long engine = (engineId != null) ? initEngine(engineId) : 0; + return new OpensslCipher(context, algMode, padding, engine); } /** Nested class for algorithm, mode and padding. */ @@ -175,7 +187,7 @@ private static Transform tokenizeTransformation(String transformation) * @param iv crypto iv */ public void init(int mode, byte[] key, byte[] iv) { - context = init(context, mode, alg, padding, key, iv); + context = init(context, mode, alg, padding, key, iv, engine); } /** @@ -255,8 +267,9 @@ public int doFinal(ByteBuffer output) throws ShortBufferException, /** Forcibly clean the context. */ public void clean() { if (context != 0) { - clean(context); + clean(context, engine); context = 0; + engine = 0; } } @@ -273,17 +286,19 @@ protected void finalize() throws Throwable { private native static void initIDs(); private native static long initContext(int alg, int padding); + + private native static long initEngine(String engineId); private native long init(long context, int mode, int alg, int padding, - byte[] key, byte[] iv); - + byte[] key, byte[] iv, long engineNum); + private native int update(long context, ByteBuffer input, int inputOffset, int inputLength, ByteBuffer output, int outputOffset, int maxOutputLength); private native int doFinal(long context, ByteBuffer output, int offset, int maxOutputLength); - private native void clean(long context); - + private native void clean(long ctx, long engineNum); + public native static String getLibraryName(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCtrCryptoCodec.java new file mode 100644 index 0000000000000..d887f15143eb9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCtrCryptoCodec.java @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.random.OpensslSecureRandom; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.util.ReflectionUtils; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.GeneralSecurityException; +import java.security.SecureRandom; +import java.util.Random; +import org.slf4j.Logger; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public abstract class OpensslCtrCryptoCodec extends CryptoCodec{ + + private Configuration conf; + private Random random; + private String engineId; + + public String getEngineId() { + return engineId; + } + + public void setEngineId(String engineId) { + this.engineId = engineId; + } + + public Random getRandom() { + return random; + } + + public void setRandom(Random random) { + this.random = random; + } + + + public void calculateIV(byte[] initIV, long counter, + byte[] iv, int blockSize) { + Preconditions.checkArgument(initIV.length == blockSize); + Preconditions.checkArgument(iv.length == blockSize); + + int i = iv.length; // IV length + int j = 0; // counter bytes index + int sum = 0; + while(i-- > 0){ + // (sum >>> Byte.SIZE) is the carry for condition + sum = (initIV[i] & 0xff) + (sum >>> Byte.SIZE); + if (j++ < 8) { // Big-endian, and long is 8 bytes length + sum += (byte) counter & 0xff; + counter >>>= 8; + } + iv[i] = (byte) sum; + } + } + + protected abstract Logger getLogger(); + + public void setConf(Configuration conf) { + this.conf = conf; + final Class klass = conf.getClass( + HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY, + OpensslSecureRandom.class, + Random.class); + try { + random = ReflectionUtils.newInstance(klass, conf); + getLogger().debug("Using " + klass.getName() + + " as random number generator."); + } catch (Exception e) { + getLogger().info("Unable to use " + klass.getName() + + ". Falling back to " + + "Java SecureRandom.", e); + this.random = new SecureRandom(); + } + } + + public Configuration getConf() { + return conf; + } + + @Override + public void generateSecureRandom(byte[] bytes) { + random.nextBytes(bytes); + } + + @Override + public void close() throws IOException { + if (this.random instanceof Closeable) { + Closeable r = (Closeable) this.random; + IOUtils.cleanupWithLogger(getLogger(), r); + } + } + + protected static class OpensslCtrCipher implements Encryptor, Decryptor { + private final OpensslCipher cipher; + private final int mode; + private boolean contextReset = false; + + public OpensslCtrCipher(int mode, CipherSuite suite, String engineId) + throws GeneralSecurityException { + this.mode = mode; + cipher = OpensslCipher.getInstance(suite.getName(), engineId); + } + + public OpensslCtrCipher(int mode, CipherSuite suite) + throws GeneralSecurityException { + this.mode = mode; + cipher = OpensslCipher.getInstance(suite.getName()); + } + + @Override + public void init(byte[] key, byte[] iv) throws IOException { + Preconditions.checkNotNull(key); + Preconditions.checkNotNull(iv); + contextReset = false; + cipher.init(mode, key, iv); + } + + /** + * AES-CTR will consume all of the input data. It requires enough space in + * the destination buffer to encrypt entire input buffer. + */ + @Override + public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) + throws IOException { + process(inBuffer, outBuffer); + } + + /** + * AES-CTR will consume all of the input data. It requires enough space in + * the destination buffer to decrypt entire input buffer. + */ + @Override + public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) + throws IOException { + process(inBuffer, outBuffer); + } + + private void process(ByteBuffer inBuffer, ByteBuffer outBuffer) + throws IOException { + try { + int inputSize = inBuffer.remaining(); + // OpensslCipher#update will maintain crypto context. + int n = cipher.update(inBuffer, outBuffer); + if (n < inputSize) { + /** + * Typically code will not get here. OpensslCipher#update will + * consume all input data and put result in outBuffer. + * OpensslCipher#doFinal will reset the crypto context. + */ + contextReset = true; + cipher.doFinal(outBuffer); + } + } catch (Exception e) { + throw new IOException(e); + } + } + + @Override + public boolean isContextReset() { + return contextReset; + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslSm4CtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslSm4CtrCryptoCodec.java new file mode 100644 index 0000000000000..f6b2f6a802556 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslSm4CtrCryptoCodec.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto; + +import org.apache.hadoop.conf.Configuration; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.security.GeneralSecurityException; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_OPENSSL_ENGINE_ID_KEY; + +/** + * Implement the SM4-CTR crypto codec using JNI into OpenSSL. + */ +@InterfaceAudience.Private +public class OpensslSm4CtrCryptoCodec extends OpensslCtrCryptoCodec { + + private static final Logger LOG = + LoggerFactory.getLogger(OpensslSm4CtrCryptoCodec.class.getName()); + + public OpensslSm4CtrCryptoCodec() { + String loadingFailureReason = OpensslCipher.getLoadingFailureReason(); + if (loadingFailureReason != null) { + throw new RuntimeException(loadingFailureReason); + } + } + + @Override + public Logger getLogger() { + return LOG; + } + + @Override + public void setConf(Configuration conf) { + super.setConf(conf); + setEngineId(conf.get(HADOOP_SECURITY_OPENSSL_ENGINE_ID_KEY)); + } + + @Override + public CipherSuite getCipherSuite() { + return CipherSuite.SM4_CTR_NOPADDING; + } + + @Override + public void calculateIV(byte[] initIV, long counter, byte[] iv) { + super.calculateIV(initIV, counter, iv, + getCipherSuite().getAlgorithmBlockSize()); + } + + @Override + public Encryptor createEncryptor() throws GeneralSecurityException { + return new OpensslCtrCipher(OpensslCipher.ENCRYPT_MODE, + getCipherSuite(), getEngineId()); + } + + @Override + public Decryptor createDecryptor() throws GeneralSecurityException { + return new OpensslCtrCipher(OpensslCipher.DECRYPT_MODE, + getCipherSuite(), getEngineId()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/CachingKeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/CachingKeyProvider.java index 7a66e1e4bab48..4f456e54a4d61 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/CachingKeyProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/CachingKeyProvider.java @@ -22,9 +22,9 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; +import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; /** * A KeyProviderExtension implementation providing a short lived diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java index 7951af56bc8f9..3c3099e113567 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java @@ -18,7 +18,7 @@ package org.apache.hadoop.crypto.key; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -32,7 +32,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import javax.crypto.spec.SecretKeySpec; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java index a8c283ab649cc..d51bf38e4ff9d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java @@ -26,12 +26,14 @@ import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.security.NoSuchAlgorithmException; +import java.security.Security; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.bouncycastle.jce.provider.BouncyCastleProvider; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonWriter; import org.apache.commons.lang3.builder.EqualsBuilder; @@ -43,6 +45,7 @@ import javax.crypto.KeyGenerator; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER; /** @@ -409,10 +412,14 @@ public KeyProvider(Configuration conf) { // java.security.UnrecoverableKeyException in JDK 8u171. if(System.getProperty(JCEKS_KEY_SERIAL_FILTER) == null) { String serialFilter = - conf.get(HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER, - JCEKS_KEY_SERIALFILTER_DEFAULT); + conf.get(HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER, + JCEKS_KEY_SERIALFILTER_DEFAULT); System.setProperty(JCEKS_KEY_SERIAL_FILTER, serialFilter); } + String jceProvider = conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY); + if (BouncyCastleProvider.PROVIDER_NAME.equals(jceProvider)) { + Security.addProvider(new BouncyCastleProvider()); + } } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java index 00d7a7dfce0f7..8aa64e2ceb6ae 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java @@ -29,7 +29,7 @@ import javax.crypto.spec.IvParameterSpec; import javax.crypto.spec.SecretKeySpec; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.crypto.CryptoCodec; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java index 05d99ed0810fc..e23d8b8e4a774 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.crypto.key; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.security.Credentials; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java index f2f6f1801c2dd..2cc011c0df34d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java @@ -25,7 +25,7 @@ import java.util.List; import java.util.Map; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.StringUtils; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java index 71ed4557b357b..bc56f0e28676b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java @@ -79,9 +79,9 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import static org.apache.hadoop.util.KMSUtil.checkNotEmpty; import static org.apache.hadoop.util.KMSUtil.checkNotNull; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java index ee2295cff77f3..237a50d0227b8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java @@ -50,8 +50,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * A simple LoadBalancing KMSClientProvider that round-robins requests @@ -502,6 +502,7 @@ public Void call(KMSClientProvider provider) throws IOException { return null; } }, nextIdx(), false); + invalidateCache(name); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java index 7d26acbf21a03..64f9ce11f27cb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java @@ -33,11 +33,11 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import com.google.common.base.Preconditions; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; +import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.classification.InterfaceAudience; /** @@ -299,19 +299,18 @@ public E getNext(String keyName) * @param keyName the key to drain the Queue for */ public void drain(String keyName) { + Runnable e; + while ((e = queue.deleteByName(keyName)) != null) { + executor.remove(e); + } + writeLock(keyName); try { - Runnable e; - while ((e = queue.deleteByName(keyName)) != null) { - executor.remove(e); - } - writeLock(keyName); - try { - keyQueues.get(keyName).clear(); - } finally { - writeUnlock(keyName); + LinkedBlockingQueue kq = keyQueues.getIfPresent(keyName); + if (kq != null) { + kq.clear(); } - } catch (ExecutionException ex) { - //NOP + } finally { + writeUnlock(keyName); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java index 1863f5ec2035f..a7a609ce440b6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.util.NativeCodeLoader; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.util.PerformanceAdvisory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OsSecureRandom.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OsSecureRandom.java index 8e191b5514e55..7dda61025f619 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OsSecureRandom.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OsSecureRandom.java @@ -24,6 +24,7 @@ import java.nio.file.Paths; import java.util.Random; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; @@ -73,7 +74,12 @@ private void fillReservoir(int min) { public OsSecureRandom() { } - + + @VisibleForTesting + public boolean isClosed() { + return stream == null; + } + @Override synchronized public void setConf(Configuration conf) { this.conf = conf; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java index ec346b482a452..5796f23026200 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java @@ -55,7 +55,7 @@ import org.apache.hadoop.util.LambdaUtils; import org.apache.hadoop.util.Progressable; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -864,6 +864,20 @@ public abstract FileStatus getFileStatus(final Path f) throws AccessControlException, FileNotFoundException, UnresolvedLinkException, IOException; + /** + * Synchronize client metadata state. + *

+ * In some FileSystem implementations such as HDFS metadata + * synchronization is essential to guarantee consistency of read requests + * particularly in HA setting. + * @throws IOException + * @throws UnsupportedOperationException + */ + public void msync() throws IOException, UnsupportedOperationException { + throw new UnsupportedOperationException(getClass().getCanonicalName() + + " does not support method msync"); + } + /** * The specification of this method matches that of * {@link FileContext#access(Path, FsAction)} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferUtil.java index c31c29b5b6d31..5708c906fe764 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferUtil.java @@ -26,7 +26,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.ByteBufferPool; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; @InterfaceAudience.Private @InterfaceStability.Evolving diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java index cc9c284c9fa55..f081742ce59db 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index ce132f9a37891..7556831701419 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -21,7 +21,9 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.crypto.JceAesCtrCryptoCodec; +import org.apache.hadoop.crypto.JceSm4CtrCryptoCodec; import org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec; +import org.apache.hadoop.crypto.OpensslSm4CtrCryptoCodec; /** * This class contains constants for configuration keys used @@ -350,6 +352,9 @@ public class CommonConfigurationKeysPublic { "hadoop.caller.context.signature.max.size"; public static final int HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT = 40; + public static final String HADOOP_CALLER_CONTEXT_SEPARATOR_KEY = + "hadoop.caller.context.separator"; + public static final String HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT = ","; /** * @see @@ -701,10 +706,18 @@ public class CommonConfigurationKeysPublic { HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY = HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX + CipherSuite.AES_CTR_NOPADDING.getConfigSuffix(); + public static final String + HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY = + HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX + + CipherSuite.SM4_CTR_NOPADDING.getConfigSuffix(); public static final String HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_DEFAULT = OpensslAesCtrCryptoCodec.class.getName() + "," + JceAesCtrCryptoCodec.class.getName(); + public static final String + HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_DEFAULT = + OpensslSm4CtrCryptoCodec.class.getName() + "," + + JceSm4CtrCryptoCodec.class.getName(); /** * @see * @@ -873,6 +886,13 @@ public class CommonConfigurationKeysPublic { * * core-default.xml */ + public static final String HADOOP_SECURITY_OPENSSL_ENGINE_ID_KEY = + "hadoop.security.openssl.engine.id"; + /** + * @see + * + * core-default.xml + */ public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY = "hadoop.security.random.device.file.path"; public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT = diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java index 7a94088e4062d..30c2faeb24d1f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java @@ -30,7 +30,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.Shell; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** Filesystem disk space usage statistics. * Uses the unix 'df' program to get mount points, and java.io.File for diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java index 6e374c97c3eda..89ac7c3e7cbbe 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.fs; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java index 2feb9375255c3..193c52c60d949 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.lang.ref.WeakReference; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java index 62a3182dfba20..7a6792817b750 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java @@ -28,7 +28,7 @@ import java.io.IOException; import java.util.EnumSet; -import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputStream.java index 672ab15f16c3b..b3b3fac0c09e1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputStream.java @@ -21,7 +21,7 @@ import java.io.IOException; import java.io.InputStream; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.slf4j.Logger; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index e5f4ef3809f18..000c158d88c2e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -65,8 +65,8 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.ShutdownHookManager; -import com.google.common.base.Preconditions; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.htrace.core.Tracer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1249,6 +1249,16 @@ public FileStatus next(final AbstractFileSystem fs, final Path p) }.resolve(this, absF); } + /** + * Synchronize client metadata state. + * + * @throws IOException + * @throws UnsupportedOperationException + */ + public void msync() throws IOException, UnsupportedOperationException { + defaultFS.msync(); + } + /** * Checks if the user can access a path. The mode specifies which access * checks to perform. If the requested permissions are granted, then the diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java index 4cfce2eed6c2c..5444103855d26 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java @@ -24,8 +24,8 @@ import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.crypto.CryptoProtocolVersion; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull; /** * FileEncryptionInfo encapsulates all the encryption-related information for diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index ab5040486dffc..9fffb8aa5b511 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -83,12 +83,12 @@ import org.apache.htrace.core.Tracer; import org.apache.htrace.core.TraceScope; -import com.google.common.base.Preconditions; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; @@ -621,6 +621,7 @@ public static LocalFileSystem newInstanceLocal(Configuration conf) * @throws IOException a problem arose closing one or more filesystem. */ public static void closeAll() throws IOException { + debugLogFileSystemClose("closeAll", ""); CACHE.closeAll(); } @@ -631,10 +632,24 @@ public static void closeAll() throws IOException { * @throws IOException a problem arose closing one or more filesystem. */ public static void closeAllForUGI(UserGroupInformation ugi) - throws IOException { + throws IOException { + debugLogFileSystemClose("closeAllForUGI", "UGI: " + ugi); CACHE.closeAll(ugi); } + private static void debugLogFileSystemClose(String methodName, + String additionalInfo) { + if (LOGGER.isDebugEnabled()) { + Throwable throwable = new Throwable().fillInStackTrace(); + LOGGER.debug("FileSystem.{}() by method: {}); {}", methodName, + throwable.getStackTrace()[2], additionalInfo); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("FileSystem.{}() full stack trace:", methodName, + throwable); + } + } + } + /** * Qualify a path to one which uses this FileSystem and, if relative, * made absolute. @@ -2214,7 +2229,9 @@ private void fetchMore() throws IOException { @Override @SuppressWarnings("unchecked") public T next() throws IOException { - Preconditions.checkState(hasNext(), "No more items in iterator"); + if (!hasNext()) { + throw new NoSuchElementException("No more items in iterator"); + } if (i == entries.getEntries().length) { fetchMore(); } @@ -2570,6 +2587,9 @@ public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) */ @Override public void close() throws IOException { + debugLogFileSystemClose("close", "Key: " + key + "; URI: " + getUri() + + "; Object Identity Hash: " + + Integer.toHexString(System.identityHashCode(this))); // delete all files that were marked as delete-on-exit. processDeleteOnExit(); CACHE.remove(this.key, this); @@ -2655,6 +2675,20 @@ public short getDefaultReplication(Path path) { */ public abstract FileStatus getFileStatus(Path f) throws IOException; + /** + * Synchronize client metadata state. + *

+ * In some FileSystem implementations such as HDFS metadata + * synchronization is essential to guarantee consistency of read requests + * particularly in HA setting. + * @throws IOException + * @throws UnsupportedOperationException + */ + public void msync() throws IOException, UnsupportedOperationException { + throw new UnsupportedOperationException(getClass().getCanonicalName() + + " does not support method msync"); + } + /** * Checks if the user can access a path. The mode specifies which access * checks to perform. If the requested permissions are granted, then the diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java index 43c23abadea44..f717e03692378 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java @@ -20,7 +20,7 @@ import java.util.Iterator; import java.util.NoSuchElementException; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileSystem.Statistics.StatisticsData; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index 7bc93f9bf5db8..e078a2c519621 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -398,6 +398,12 @@ public static boolean copy(FileSystem srcFS, FileStatus srcStatus, Configuration conf) throws IOException { Path src = srcStatus.getPath(); dst = checkDest(src.getName(), dstFS, dst, overwrite); + + if (srcFS.makeQualified(src).equals(dstFS.makeQualified(dst))) { + throw new PathOperationException("Source (" + src + ") and destination " + + "(" + dst + ") are equal in the copy command."); + } + if (srcStatus.isDirectory()) { checkDependencies(srcFS, src, dstFS, dst); if (!dstFS.mkdirs(dst)) { @@ -1806,7 +1812,7 @@ public static FileSystem write(final FileSystem fs, final Path path, * specified charset. This utility method opens the file for writing, creating * the file if it does not exist, or overwrites an existing file. * - * @param FileContext the file context with which to create the file + * @param fs the file context with which to create the file * @param path the path to the file * @param charseq the char sequence to write to the file * @param cs the charset to use for encoding diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index 42410974db17c..607aa263622f6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -462,6 +462,11 @@ public FileStatus getFileStatus(Path f) throws IOException { return fs.getFileStatus(f); } + @Override + public void msync() throws IOException, UnsupportedOperationException { + fs.msync(); + } + @Override public void access(Path path, FsAction mode) throws AccessControlException, FileNotFoundException, IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java index 27e75d8a25df6..7d979b37b4a50 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java @@ -124,6 +124,11 @@ public FileStatus getFileStatus(Path f) return myFs.getFileStatus(f); } + @Override + public void msync() throws IOException, UnsupportedOperationException { + myFs.msync(); + } + @Override public void access(Path path, FsAction mode) throws AccessControlException, FileNotFoundException, UnresolvedLinkException, IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsConstants.java index 344048f0ceeb1..603454210644d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsConstants.java @@ -45,5 +45,4 @@ public interface FsConstants { String FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN = "fs.viewfs.overload.scheme.target.%s.impl"; String VIEWFS_TYPE = "viewfs"; - String VIEWFSOS_TYPE = "viewfsOverloadScheme"; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java index f6396482d18c1..43d2f3ef19a6c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java @@ -56,6 +56,7 @@ public Writable newInstance() { private DataChecksum.Type checksumType; private String keyProviderUri; private byte storagepolicyId; + private boolean snapshotTrashRootEnabled; public FsServerDefaults() { } @@ -83,6 +84,18 @@ public FsServerDefaults(long blockSize, int bytesPerChecksum, boolean encryptDataTransfer, long trashInterval, DataChecksum.Type checksumType, String keyProviderUri, byte storagepolicy) { + this(blockSize, bytesPerChecksum, writePacketSize, replication, + fileBufferSize, encryptDataTransfer, trashInterval, + checksumType, keyProviderUri, storagepolicy, + false); + } + + public FsServerDefaults(long blockSize, int bytesPerChecksum, + int writePacketSize, short replication, int fileBufferSize, + boolean encryptDataTransfer, long trashInterval, + DataChecksum.Type checksumType, + String keyProviderUri, byte storagepolicy, + boolean snapshotTrashRootEnabled) { this.blockSize = blockSize; this.bytesPerChecksum = bytesPerChecksum; this.writePacketSize = writePacketSize; @@ -93,6 +106,7 @@ public FsServerDefaults(long blockSize, int bytesPerChecksum, this.checksumType = checksumType; this.keyProviderUri = keyProviderUri; this.storagepolicyId = storagepolicy; + this.snapshotTrashRootEnabled = snapshotTrashRootEnabled; } public long getBlockSize() { @@ -139,6 +153,10 @@ public byte getDefaultStoragePolicyId() { return storagepolicyId; } + public boolean getSnapshotTrashRootEnabled() { + return snapshotTrashRootEnabled; + } + // ///////////////////////////////////////// // Writable // ///////////////////////////////////////// diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsTracer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsTracer.java index e422336739a44..6fab4bdfebc99 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsTracer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsTracer.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.fs; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsUrlConnection.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsUrlConnection.java index c5429d2370250..11b3e91e86c3a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsUrlConnection.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsUrlConnection.java @@ -24,7 +24,7 @@ import java.net.URL; import java.net.URLConnection; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobalStorageStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobalStorageStatistics.java index 2dba525e5d9d1..0cf3b62bccfe9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobalStorageStatistics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobalStorageStatistics.java @@ -23,7 +23,7 @@ import java.util.NoSuchElementException; import java.util.TreeMap; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java index f301f22057925..9cdcb4ac4acd3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java @@ -32,7 +32,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull; /** * Implementation of {@link FileSystem#globStatus(Path, PathFilter)}. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java index 5f4c4a236e96c..02c0916e54447 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java @@ -676,6 +676,11 @@ private HarStatus getFileHarStatus(Path f) throws IOException { return hstatus; } + @Override + public void msync() throws IOException, UnsupportedOperationException { + fs.msync(); + } + /** * @return null since no checksum algorithm is implemented. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java index 8b47dfeb9a7ce..30f793dadfec3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java @@ -29,7 +29,7 @@ import org.apache.hadoop.util.Shell.ExitCodeException; import org.apache.hadoop.util.Shell.ShellCommandExecutor; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import static java.nio.file.Files.createLink; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PartialListing.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PartialListing.java index 043f84612dc8b..cec5d68341f92 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PartialListing.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PartialListing.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.fs; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -30,7 +30,7 @@ * A partial listing of the children of a parent directory. Since it is a * partial listing, multiple PartialListing may need to be combined to obtain * the full listing of a parent directory. - *

+ *

* ListingBatch behaves similar to a Future, in that getting the result via * {@link #get()} will throw an Exception if there was a failure. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index 72eeb99a4ea5d..ba29f74cc5ca4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -19,7 +19,7 @@ package org.apache.hadoop.fs; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.io.BufferedOutputStream; import java.io.DataOutput; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java index f6c2f2af1c9ba..11e827688a294 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java @@ -29,7 +29,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Shell; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Wrapper for the Unix stat(1) command. Used to workaround the lack of diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java index 2ecd206a04080..1f67e540cd29c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java @@ -34,13 +34,15 @@ @InterfaceStability.Unstable public enum StorageType { // sorted by the speed of the storage types, from fast to slow - RAM_DISK(true), - SSD(false), - DISK(false), - ARCHIVE(false), - PROVIDED(false); + RAM_DISK(true, true), + NVDIMM(false, true), + SSD(false, false), + DISK(false, false), + ARCHIVE(false, false), + PROVIDED(false, false); private final boolean isTransient; + private final boolean isRAM; public static final StorageType DEFAULT = DISK; @@ -48,14 +50,19 @@ public enum StorageType { private static final StorageType[] VALUES = values(); - StorageType(boolean isTransient) { + StorageType(boolean isTransient, boolean isRAM) { this.isTransient = isTransient; + this.isRAM = isRAM; } public boolean isTransient() { return isTransient; } + public boolean isRAM() { + return isRAM; + } + public boolean supportTypeQuota() { return !isTransient; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java index 18972ea3ecf79..7682992d42590 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java @@ -38,7 +38,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Time; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/UnionStorageStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/UnionStorageStatistics.java index 3d5b6af794682..2497ded48e7e9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/UnionStorageStatistics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/UnionStorageStatistics.java @@ -20,7 +20,7 @@ import java.util.Iterator; import java.util.NoSuchElementException; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/XAttrCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/XAttrCodec.java index e15968dd6d273..bfd4daffc7cad 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/XAttrCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/XAttrCodec.java @@ -25,7 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * The value of XAttr is byte[], this class is to diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java index 28db2c9a1a227..6899bb8d87426 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java @@ -24,8 +24,8 @@ import java.net.ConnectException; import java.net.URI; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.net.ftp.FTP; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPFile; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractFSBuilderImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractFSBuilderImpl.java index 5fc92e97be76c..9cf8b3dc4d203 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractFSBuilderImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractFSBuilderImpl.java @@ -26,7 +26,7 @@ import java.util.Optional; import java.util.Set; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -36,8 +36,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathHandle; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull; /** * Builder for filesystem/filecontext operations of various kinds, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java index d8b7fe0744087..ed4bcc84e96f9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java @@ -24,14 +24,14 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.fs.MultipartUploader; import org.apache.hadoop.fs.PartHandle; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UploadHandle; -import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument; /** * Standard base class for Multipart Uploaders. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java index ae0def0e378d4..7c5a5d949a072 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java @@ -30,8 +30,8 @@ import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -import com.google.common.base.Charsets; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FsLinkResolution.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FsLinkResolution.java index f5ef8c4923328..8d4bebda15096 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FsLinkResolution.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FsLinkResolution.java @@ -20,7 +20,7 @@ import java.io.IOException; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java index f13d701803d7e..84ca94e642833 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java @@ -166,11 +166,11 @@ private static IOException unwrapInnerException(final Throwable e) { * Propagate options to any builder, converting everything with the * prefix to an option where, if there were 2+ dot-separated elements, * it is converted to a schema. - * 

+   * 
{@code
    *   fs.example.s3a.option => s3a:option
    *   fs.example.fs.io.policy => s3a.io.policy
    *   fs.example.something => something
-   * 

+   * }
* @param builder builder to modify * @param conf configuration to read * @param optionalPrefix prefix for optional settings @@ -196,11 +196,11 @@ FSBuilder propagateOptions( * Propagate options to any builder, converting everything with the * prefix to an option where, if there were 2+ dot-separated elements, * it is converted to a schema. - * 
+   * 
{@code
    *   fs.example.s3a.option => s3a:option
    *   fs.example.fs.io.policy => s3a.io.policy
    *   fs.example.something => something
-   * 

+   * }
* @param builder builder to modify * @param conf configuration to read * @param prefix prefix to scan/strip diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/MultipartUploaderBuilderImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/MultipartUploaderBuilderImpl.java index 6c3336e6882b3..88c573acc4cb3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/MultipartUploaderBuilderImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/MultipartUploaderBuilderImpl.java @@ -34,7 +34,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java index 9332ac6e7eedb..1e3e43581dccc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java @@ -25,7 +25,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathCapabilities; -import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument; @InterfaceAudience.Private @InterfaceStability.Evolving diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java index 1de1ecb785368..ae1d548d34dfc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java @@ -21,7 +21,7 @@ import java.io.IOException; import java.util.concurrent.ExecutionException; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntry.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntry.java index a902488377239..04a045299b7ea 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntry.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntry.java @@ -21,7 +21,7 @@ import java.util.Collection; import java.util.List; -import com.google.common.base.Objects; +import org.apache.hadoop.thirdparty.com.google.common.base.Objects; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java index 385fed21d4194..674b88083d3dc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java @@ -22,9 +22,9 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Objects; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * An AclStatus contains the ACL information of a specific file. AclStatus diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java index 42492520dceaa..58b24f200429b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java @@ -23,7 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * AclUtil contains utility methods for manipulating ACLs. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java index a91b50f2e9fa7..898f615ec4633 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java @@ -34,7 +34,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import com.jcraft.jsch.ChannelSftp; import com.jcraft.jsch.ChannelSftp.LsEntry; import com.jcraft.jsch.SftpATTRS; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java index 5a60ef2ae9b03..dcff0094eccf5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java @@ -22,7 +22,7 @@ import java.util.LinkedList; import java.util.List; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java index ca9961aeb65a0..90a709dffc0c1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java @@ -54,6 +54,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; import static org.apache.hadoop.fs.CreateFlag.CREATE; import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST; +import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; /** * Provides: argument processing to ensure the destination is valid @@ -515,7 +516,8 @@ FSDataOutputStream create(PathData item, boolean lazyPersist) defaultBlockSize = getDefaultBlockSize(item.path); } - EnumSet createFlags = EnumSet.of(CREATE, LAZY_PERSIST); + EnumSet createFlags = + EnumSet.of(CREATE, LAZY_PERSIST, OVERWRITE); return create(item.path, FsPermission.getFileDefault().applyUMask( FsPermission.getUMask(getConf())), diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Concat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Concat.java new file mode 100644 index 0000000000000..f25b689e7ed17 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Concat.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.shell; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.LinkedList; + +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathIOException; + +/** + * Concat the given files. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class Concat extends FsCommand { + public static void registerCommands(CommandFactory factory) { + factory.addClass(Concat.class, "-concat"); + } + + public static final String NAME = "concat"; + public static final String USAGE = " ..."; + public static final String DESCRIPTION = "Concatenate existing source files" + + " into the target file. Target file and source files should be in the" + + " same directory."; + private static FileSystem testFs; // test only. + + @Override + protected void processArguments(LinkedList args) + throws IOException { + if (args.size() < 1) { + throw new IOException("Target path not specified. " + USAGE); + } + if (args.size() < 3) { + throw new IOException( + "The number of source paths is less than 2. " + USAGE); + } + PathData target = args.removeFirst(); + LinkedList srcList = args; + if (!target.exists || !target.stat.isFile()) { + throw new FileNotFoundException(String + .format("Target path %s does not exist or is" + " not file.", + target.path)); + } + Path[] srcArray = new Path[srcList.size()]; + for (int i = 0; i < args.size(); i++) { + PathData src = srcList.get(i); + if (!src.exists || !src.stat.isFile()) { + throw new FileNotFoundException( + String.format("%s does not exist or is not file.", src.path)); + } + srcArray[i] = src.path; + } + FileSystem fs = target.fs; + if (testFs != null) { + fs = testFs; + } + try { + fs.concat(target.path, srcArray); + } catch (UnsupportedOperationException exception) { + throw new PathIOException("Dest filesystem '" + fs.getUri().getScheme() + + "' doesn't support concat.", exception); + } + } + + @VisibleForTesting + static void setTestFs(FileSystem fs) { + testFs = fs; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java index 39958a9cb1c9d..77f63170593ab 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java @@ -30,7 +30,7 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.TimeUnit; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FSDataInputStream; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java index ab7e1951bcd3f..cb3858d63b2ec 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java @@ -89,7 +89,7 @@ public static void registerCommands(CommandFactory factory) { "Otherwise, it displays the quota and usage for all the storage \n" + "types that support quota. The list of possible storage " + "types(case insensitive):\n" + - "ram_disk, ssd, disk and archive.\n" + + "ram_disk, ssd, disk, archive and nvdimm.\n" + "It can also pass the value '', 'all' or 'ALL' to specify all " + "the storage types.\n" + "The -" + OPTION_QUOTA_AND_USAGE + " option shows the quota and \n" + diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java index ea8378dc4551b..0abf0e3c587ad 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java @@ -70,6 +70,7 @@ public static void registerCommands(CommandFactory factory) { factory.registerCommands(Truncate.class); factory.registerCommands(SnapshotCommands.class); factory.registerCommands(XAttrCommands.class); + factory.registerCommands(Concat.class); } protected FsCommand() {} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java index efc541ccf81ee..b50eb69a26d70 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java @@ -25,7 +25,7 @@ import java.util.Date; import java.util.LinkedList; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SnapshotCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SnapshotCommands.java index 4bd596a40d678..75dc86ec87c18 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SnapshotCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SnapshotCommands.java @@ -26,7 +26,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIsNotDirectoryException; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Snapshot related operations diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java index 8a75a60f435ca..22dd32bce8512 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java @@ -28,7 +28,7 @@ import org.apache.hadoop.fs.PathIsDirectoryException; import org.apache.hadoop.io.IOUtils; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Get a listing of all files in that match the file patterns. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java index be174b5e9cf68..b81f2f7b97575 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java @@ -31,7 +31,7 @@ import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.util.StringUtils; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Unix touch like commands @@ -137,7 +137,7 @@ protected void processOptions(LinkedList args) { CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, OPTION_USE_TIMESTAMP, OPTION_CHANGE_ONLY_ACCESS_TIME, - OPTION_CHANGE_ONLY_MODIFICATION_TIME); + OPTION_CHANGE_ONLY_MODIFICATION_TIME, OPTION_DO_NOT_CREATE_FILE); cf.parse(args); this.changeModTime = cf.getOpt(OPTION_CHANGE_ONLY_MODIFICATION_TIME); this.changeAccessTime = cf.getOpt(OPTION_CHANGE_ONLY_ACCESS_TIME); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java index 630177611940e..2fe7c858e4e66 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java @@ -23,7 +23,7 @@ import java.util.Map; import java.util.Map.Entry; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java index 7d29b8f44ca62..09ec5d2933058 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java @@ -166,6 +166,28 @@ public static void addLinkNfly(final Configuration conf, final String src, addLinkNfly(conf, getDefaultMountTableName(conf), src, null, targets); } + /** + * Add a LinkRegex to the config for the specified mount table. + * @param conf - get mountable config from this conf + * @param mountTableName - the mountable name of the regex config item + * @param srcRegex - the src path regex expression that applies to this config + * @param targetStr - the string of target path + * @param interceptorSettings - the serialized interceptor string to be + * applied while resolving the mapping + */ + public static void addLinkRegex( + Configuration conf, final String mountTableName, final String srcRegex, + final String targetStr, final String interceptorSettings) { + String prefix = getConfigViewFsPrefix(mountTableName) + "." + + Constants.CONFIG_VIEWFS_LINK_REGEX + "."; + if ((interceptorSettings != null) && (!interceptorSettings.isEmpty())) { + prefix = prefix + interceptorSettings + + RegexMountPoint.SETTING_SRCREGEX_SEP; + } + String key = prefix + srcRegex; + conf.set(key, targetStr); + } + /** * Add config variable for homedir for default mount table * @param conf - add to this conf diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java index 492cb87ee024e..5c27692eb53fd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java @@ -86,6 +86,14 @@ public interface Constants { */ String CONFIG_VIEWFS_LINK_MERGE_SLASH = "linkMergeSlash"; + /** + * Config variable for specifying a regex link which uses regular expressions + * as source and target could use group captured in src. + * E.g. {@literal (^/(?\\w+), /prefix-${firstDir}) => + * (/path1/file1 => /prefix-path1/file1)} + */ + String CONFIG_VIEWFS_LINK_REGEX = "linkRegex"; + FsPermission PERMISSION_555 = new FsPermission((short) 0555); String CONFIG_VIEWFS_RENAME_STRATEGY = "fs.viewfs.rename.strategy"; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/HCFSMountTableConfigLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/HCFSMountTableConfigLoader.java index 3968e3650cf39..8dbb0f3007233 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/HCFSMountTableConfigLoader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/HCFSMountTableConfigLoader.java @@ -42,7 +42,7 @@ public class HCFSMountTableConfigLoader implements MountTableConfigLoader { * Loads the mount-table configuration from hadoop compatible file system and * add the configuration items to given configuration. Mount-table * configuration format should be suffixed with version number. - * Format: mount-table..xml + * Format: {@literal mount-table..xml} * Example: mount-table.1.xml * When user wants to update mount-table, the expectation is to upload new * mount-table configuration file with monotonically increasing integer as diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java index 422e7337b57fb..fd7b5619b274a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.fs.viewfs; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; @@ -39,6 +39,8 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * InodeTree implements a mount-table as a tree of inodes. @@ -46,19 +48,22 @@ * In order to use it the caller must subclass it and implement * the abstract methods {@link #getTargetFileSystem(INodeDir)}, etc. * - * The mountable is initialized from the config variables as + * The mountable is initialized from the config variables as * specified in {@link ViewFs} * * @param is AbstractFileSystem or FileSystem * * The two main methods are * {@link #InodeTree(Configuration, String)} // constructor - * {@link #resolve(String, boolean)} + * {@link #resolve(String, boolean)} */ @InterfaceAudience.Private @InterfaceStability.Unstable abstract class InodeTree { + private static final Logger LOGGER = + LoggerFactory.getLogger(InodeTree.class.getName()); + enum ResultKind { INTERNAL_DIR, EXTERNAL_DIR @@ -72,6 +77,8 @@ enum ResultKind { // the homedir for this mount table private final String homedirPrefix; private List> mountPoints = new ArrayList>(); + private List> regexMountPointList = + new ArrayList>(); static class MountPoint { String src; @@ -122,7 +129,7 @@ boolean isLink() { */ static class INodeDir extends INode { private final Map> children = new HashMap<>(); - private T internalDirFs = null; //filesystem of this internal directory + private T internalDirFs = null; //filesystem of this internal directory private boolean isRoot = false; private INodeLink fallbackLink = null; @@ -226,7 +233,14 @@ enum LinkType { * Config prefix: fs.viewfs.mounttable..linkNfly * Refer: {@link Constants#CONFIG_VIEWFS_LINK_NFLY} */ - NFLY; + NFLY, + /** + * Link entry which source are regex exrepssions and target refer matched + * group from source + * Config prefix: fs.viewfs.mounttable..linkRegex + * Refer: {@link Constants#CONFIG_VIEWFS_LINK_REGEX} + */ + REGEX; } /** @@ -394,6 +408,15 @@ private boolean hasFallbackLink() { return rootFallbackLink != null; } + /** + * @return true if the root represented as internalDir. In LinkMergeSlash, + * there will be root to root mapping. So, root does not represent as + * internalDir. + */ + protected boolean isRootInternalDir() { + return root.isInternalDir(); + } + protected INodeLink getRootFallbackLink() { Preconditions.checkState(root.isInternalDir()); return rootFallbackLink; @@ -453,7 +476,7 @@ Configuration getConfig() { /** * Create Inode Tree from the specified mount-table specified in Config - * @param config - the mount table keys are prefixed with + * @param config - the mount table keys are prefixed with * FsConstants.CONFIG_VIEWFS_PREFIX * @param viewName - the name of the mount table - if null use defaultMT name * @throws UnsupportedFileSystemException @@ -486,84 +509,82 @@ protected InodeTree(final Configuration config, final String viewName, final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); for (Entry si : config) { final String key = si.getKey(); - if (key.startsWith(mountTablePrefix)) { - gotMountTableEntry = true; - LinkType linkType; - String src = key.substring(mountTablePrefix.length()); - String settings = null; - if (src.startsWith(linkPrefix)) { - src = src.substring(linkPrefix.length()); - if (src.equals(SlashPath.toString())) { - throw new UnsupportedFileSystemException("Unexpected mount table " - + "link entry '" + key + "'. Use " - + Constants.CONFIG_VIEWFS_LINK_MERGE_SLASH + " instead!"); - } - linkType = LinkType.SINGLE; - } else if (src.startsWith(linkFallbackPrefix)) { - if (src.length() != linkFallbackPrefix.length()) { - throw new IOException("ViewFs: Mount points initialization error." + - " Invalid " + Constants.CONFIG_VIEWFS_LINK_FALLBACK + - " entry in config: " + src); - } - linkType = LinkType.SINGLE_FALLBACK; - } else if (src.startsWith(linkMergePrefix)) { // A merge link - src = src.substring(linkMergePrefix.length()); - linkType = LinkType.MERGE; - } else if (src.startsWith(linkMergeSlashPrefix)) { - // This is a LinkMergeSlash entry. This entry should - // not have any additional source path. - if (src.length() != linkMergeSlashPrefix.length()) { - throw new IOException("ViewFs: Mount points initialization error." + - " Invalid " + Constants.CONFIG_VIEWFS_LINK_MERGE_SLASH + - " entry in config: " + src); - } - linkType = LinkType.MERGE_SLASH; - } else if (src.startsWith(Constants.CONFIG_VIEWFS_LINK_NFLY)) { - // prefix.settings.src - src = src.substring(Constants.CONFIG_VIEWFS_LINK_NFLY.length() + 1); - // settings.src - settings = src.substring(0, src.indexOf('.')); - // settings - - // settings.src - src = src.substring(settings.length() + 1); - // src - - linkType = LinkType.NFLY; - } else if (src.startsWith(Constants.CONFIG_VIEWFS_HOMEDIR)) { - // ignore - we set home dir from config - continue; - } else { - throw new IOException("ViewFs: Cannot initialize: Invalid entry in " + - "Mount table in config: " + src); + if (!key.startsWith(mountTablePrefix)) { + continue; + } + + gotMountTableEntry = true; + LinkType linkType; + String src = key.substring(mountTablePrefix.length()); + String settings = null; + if (src.startsWith(linkPrefix)) { + src = src.substring(linkPrefix.length()); + if (src.equals(SlashPath.toString())) { + throw new UnsupportedFileSystemException("Unexpected mount table " + + "link entry '" + key + "'. Use " + + Constants.CONFIG_VIEWFS_LINK_MERGE_SLASH + " instead!"); } + linkType = LinkType.SINGLE; + } else if (src.startsWith(linkFallbackPrefix)) { + checkMntEntryKeyEqualsTarget(src, linkFallbackPrefix); + linkType = LinkType.SINGLE_FALLBACK; + } else if (src.startsWith(linkMergePrefix)) { // A merge link + src = src.substring(linkMergePrefix.length()); + linkType = LinkType.MERGE; + } else if (src.startsWith(linkMergeSlashPrefix)) { + // This is a LinkMergeSlash entry. This entry should + // not have any additional source path. + checkMntEntryKeyEqualsTarget(src, linkMergeSlashPrefix); + linkType = LinkType.MERGE_SLASH; + } else if (src.startsWith(Constants.CONFIG_VIEWFS_LINK_NFLY)) { + // prefix.settings.src + src = src.substring(Constants.CONFIG_VIEWFS_LINK_NFLY.length() + 1); + // settings.src + settings = src.substring(0, src.indexOf('.')); + // settings + + // settings.src + src = src.substring(settings.length() + 1); + // src + + linkType = LinkType.NFLY; + } else if (src.startsWith(Constants.CONFIG_VIEWFS_LINK_REGEX)) { + linkEntries.add( + buildLinkRegexEntry(config, ugi, src, si.getValue())); + continue; + } else if (src.startsWith(Constants.CONFIG_VIEWFS_HOMEDIR)) { + // ignore - we set home dir from config + continue; + } else { + throw new IOException("ViewFs: Cannot initialize: Invalid entry in " + + "Mount table in config: " + src); + } - final String target = si.getValue(); - if (linkType != LinkType.MERGE_SLASH) { - if (isMergeSlashConfigured) { - throw new IOException("Mount table " + mountTableName - + " has already been configured with a merge slash link. " - + "A regular link should not be added."); - } - linkEntries.add( - new LinkEntry(src, target, linkType, settings, ugi, config)); - } else { - if (!linkEntries.isEmpty()) { - throw new IOException("Mount table " + mountTableName - + " has already been configured with regular links. " - + "A merge slash link should not be configured."); - } - if (isMergeSlashConfigured) { - throw new IOException("Mount table " + mountTableName - + " has already been configured with a merge slash link. " - + "Multiple merge slash links for the same mount table is " - + "not allowed."); - } - isMergeSlashConfigured = true; - mergeSlashTarget = target; + final String target = si.getValue(); + if (linkType != LinkType.MERGE_SLASH) { + if (isMergeSlashConfigured) { + throw new IOException("Mount table " + mountTableName + + " has already been configured with a merge slash link. " + + "A regular link should not be added."); + } + linkEntries.add( + new LinkEntry(src, target, linkType, settings, ugi, config)); + } else { + if (!linkEntries.isEmpty()) { + throw new IOException("Mount table " + mountTableName + + " has already been configured with regular links. " + + "A merge slash link should not be configured."); } + if (isMergeSlashConfigured) { + throw new IOException("Mount table " + mountTableName + + " has already been configured with a merge slash link. " + + "Multiple merge slash links for the same mount table is " + + "not allowed."); + } + isMergeSlashConfigured = true; + mergeSlashTarget = target; } - } + } // End of for loop. if (isMergeSlashConfigured) { Preconditions.checkNotNull(mergeSlashTarget); @@ -578,7 +599,8 @@ protected InodeTree(final Configuration config, final String viewName, getRootDir().setRoot(true); INodeLink fallbackLink = null; for (LinkEntry le : linkEntries) { - if (le.isLinkType(LinkType.SINGLE_FALLBACK)) { + switch (le.getLinkType()) { + case SINGLE_FALLBACK: if (fallbackLink != null) { throw new IOException("Mount table " + mountTableName + " has already been configured with a link fallback. " @@ -588,7 +610,11 @@ protected InodeTree(final Configuration config, final String viewName, fallbackLink = new INodeLink(mountTableName, ugi, getTargetFileSystem(new URI(le.getTarget())), new URI(le.getTarget())); - } else { + continue; + case REGEX: + addRegexMountEntry(le); + continue; + default: createLink(le.getSrc(), le.getTarget(), le.getLinkType(), le.getSettings(), le.getUgi(), le.getConfig()); } @@ -599,9 +625,10 @@ protected InodeTree(final Configuration config, final String viewName, if (!gotMountTableEntry) { if (!initingUriAsFallbackOnNoMounts) { - throw new IOException( - "ViewFs: Cannot initialize: Empty Mount table in config for " - + "viewfs://" + mountTableName + "/"); + throw new IOException(new StringBuilder( + "ViewFs: Cannot initialize: Empty Mount table in config for ") + .append(theUri.getScheme()).append("://").append(mountTableName) + .append("/").toString()); } StringBuilder msg = new StringBuilder("Empty mount table detected for ").append(theUri) @@ -614,6 +641,55 @@ protected InodeTree(final Configuration config, final String viewName, } } + private void checkMntEntryKeyEqualsTarget( + String mntEntryKey, String targetMntEntryKey) throws IOException { + if (!mntEntryKey.equals(targetMntEntryKey)) { + throw new IOException("ViewFs: Mount points initialization error." + + " Invalid " + targetMntEntryKey + + " entry in config: " + mntEntryKey); + } + } + + private void addRegexMountEntry(LinkEntry le) throws IOException { + LOGGER.info("Add regex mount point:" + le.getSrc() + + ", target:" + le.getTarget() + + ", interceptor settings:" + le.getSettings()); + RegexMountPoint regexMountPoint = + new RegexMountPoint( + this, le.getSrc(), le.getTarget(), le.getSettings()); + regexMountPoint.initialize(); + regexMountPointList.add(regexMountPoint); + } + + private LinkEntry buildLinkRegexEntry( + Configuration config, UserGroupInformation ugi, + String mntEntryStrippedKey, String mntEntryValue) { + String linkKeyPath = null; + String settings = null; + final String linkRegexPrefix = Constants.CONFIG_VIEWFS_LINK_REGEX + "."; + // settings#.linkKey + String settingsAndLinkKeyPath = + mntEntryStrippedKey.substring(linkRegexPrefix.length()); + int settingLinkKeySepIndex = settingsAndLinkKeyPath + .indexOf(RegexMountPoint.SETTING_SRCREGEX_SEP); + if (settingLinkKeySepIndex == -1) { + // There's no settings + linkKeyPath = settingsAndLinkKeyPath; + settings = null; + } else { + // settings#.linkKey style configuration + // settings from settings#.linkKey + settings = + settingsAndLinkKeyPath.substring(0, settingLinkKeySepIndex); + // linkKeyPath + linkKeyPath = settingsAndLinkKeyPath.substring( + settings.length() + RegexMountPoint.SETTING_SRCREGEX_SEP + .length()); + } + return new LinkEntry( + linkKeyPath, mntEntryValue, LinkType.REGEX, settings, ugi, config); + } + /** * Resolve returns ResolveResult. * The caller can continue the resolution of the remainingPath @@ -630,23 +706,31 @@ static class ResolveResult { final T targetFileSystem; final String resolvedPath; final Path remainingPath; // to resolve in the target FileSystem + private final boolean isLastInternalDirLink; ResolveResult(final ResultKind k, final T targetFs, final String resolveP, - final Path remainingP) { + final Path remainingP, boolean isLastIntenalDirLink) { kind = k; targetFileSystem = targetFs; resolvedPath = resolveP; remainingPath = remainingP; + this.isLastInternalDirLink = isLastIntenalDirLink; } // Internal dir path resolution completed within the mount table boolean isInternalDir() { return (kind == ResultKind.INTERNAL_DIR); } + + // Indicates whether the internal dir path resolution completed at the link + // or resolved due to fallback. + boolean isLastInternalDirLink() { + return this.isLastInternalDirLink; + } } /** - * Resolve the pathname p relative to root InodeDir + * Resolve the pathname p relative to root InodeDir. * @param p - input path * @param resolveLastComponent * @return ResolveResult which allows further resolution of the remaining path @@ -654,13 +738,15 @@ boolean isInternalDir() { */ ResolveResult resolve(final String p, final boolean resolveLastComponent) throws FileNotFoundException { + ResolveResult resolveResult = null; String[] path = breakIntoPathComponents(p); if (path.length <= 1) { // special case for when path is "/" T targetFs = root.isInternalDir() ? - getRootDir().getInternalDirFs() : getRootLink().getTargetFileSystem(); - ResolveResult res = new ResolveResult(ResultKind.INTERNAL_DIR, - targetFs, root.fullPath, SlashPath); - return res; + getRootDir().getInternalDirFs() + : getRootLink().getTargetFileSystem(); + resolveResult = new ResolveResult(ResultKind.INTERNAL_DIR, + targetFs, root.fullPath, SlashPath, false); + return resolveResult; } /** @@ -676,22 +762,30 @@ ResolveResult resolve(final String p, final boolean resolveLastComponent) remainingPathStr.append("/").append(path[i]); } remainingPath = new Path(remainingPathStr.toString()); - ResolveResult res = new ResolveResult(ResultKind.EXTERNAL_DIR, - getRootLink().getTargetFileSystem(), root.fullPath, remainingPath); - return res; + resolveResult = new ResolveResult(ResultKind.EXTERNAL_DIR, + getRootLink().getTargetFileSystem(), root.fullPath, remainingPath, + true); + return resolveResult; } Preconditions.checkState(root.isInternalDir()); INodeDir curInode = getRootDir(); + // Try to resolve path in the regex mount point + resolveResult = tryResolveInRegexMountpoint(p, resolveLastComponent); + if (resolveResult != null) { + return resolveResult; + } + int i; // ignore first slash for (i = 1; i < path.length - (resolveLastComponent ? 0 : 1); i++) { INode nextInode = curInode.resolveInternal(path[i]); if (nextInode == null) { if (hasFallbackLink()) { - return new ResolveResult(ResultKind.EXTERNAL_DIR, - getRootFallbackLink().getTargetFileSystem(), - root.fullPath, new Path(p)); + resolveResult = new ResolveResult(ResultKind.EXTERNAL_DIR, + getRootFallbackLink().getTargetFileSystem(), root.fullPath, + new Path(p), false); + return resolveResult; } else { StringBuilder failedAt = new StringBuilder(path[0]); for (int j = 1; j <= i; ++j) { @@ -708,16 +802,17 @@ ResolveResult resolve(final String p, final boolean resolveLastComponent) if (i >= path.length - 1) { remainingPath = SlashPath; } else { - StringBuilder remainingPathStr = new StringBuilder("/" + path[i + 1]); + StringBuilder remainingPathStr = + new StringBuilder("/" + path[i + 1]); for (int j = i + 2; j < path.length; ++j) { remainingPathStr.append('/').append(path[j]); } remainingPath = new Path(remainingPathStr.toString()); } - final ResolveResult res = - new ResolveResult(ResultKind.EXTERNAL_DIR, - link.getTargetFileSystem(), nextInode.fullPath, remainingPath); - return res; + resolveResult = new ResolveResult(ResultKind.EXTERNAL_DIR, + link.getTargetFileSystem(), nextInode.fullPath, remainingPath, + true); + return resolveResult; } else if (nextInode.isInternalDir()) { curInode = (INodeDir) nextInode; } @@ -738,10 +833,77 @@ ResolveResult resolve(final String p, final boolean resolveLastComponent) } remainingPath = new Path(remainingPathStr.toString()); } - final ResolveResult res = - new ResolveResult(ResultKind.INTERNAL_DIR, - curInode.getInternalDirFs(), curInode.fullPath, remainingPath); - return res; + resolveResult = new ResolveResult(ResultKind.INTERNAL_DIR, + curInode.getInternalDirFs(), curInode.fullPath, remainingPath, false); + return resolveResult; + } + + /** + * Walk through all regex mount points to see + * whether the path match any regex expressions. + * E.g. link: ^/user/(?\\w+) => s3://$user.apache.com/_${user} + * srcPath: is /user/hadoop/dir1 + * resolveLastComponent: true + * then return value is s3://hadoop.apache.com/_hadoop + * + * @param srcPath + * @param resolveLastComponent + * @return + */ + protected ResolveResult tryResolveInRegexMountpoint(final String srcPath, + final boolean resolveLastComponent) { + for (RegexMountPoint regexMountPoint : regexMountPointList) { + ResolveResult resolveResult = + regexMountPoint.resolve(srcPath, resolveLastComponent); + if (resolveResult != null) { + return resolveResult; + } + } + return null; + } + + /** + * Build resolve result. + * Here's an example + * Mountpoint: fs.viewfs.mounttable.mt + * .linkRegex.replaceresolveddstpath:_:-#.^/user/(?\w+) + * Value: /targetTestRoot/$username + * Dir path to test: + * viewfs://mt/user/hadoop_user1/hadoop_dir1 + * Expect path: /targetTestRoot/hadoop-user1/hadoop_dir1 + * resolvedPathStr: /user/hadoop_user1 + * targetOfResolvedPathStr: /targetTestRoot/hadoop-user1 + * remainingPath: /hadoop_dir1 + * + * @return targetFileSystem or null on exceptions. + */ + protected ResolveResult buildResolveResultForRegexMountPoint( + ResultKind resultKind, String resolvedPathStr, + String targetOfResolvedPathStr, Path remainingPath) { + try { + T targetFs = getTargetFileSystem( + new URI(targetOfResolvedPathStr)); + return new ResolveResult(resultKind, targetFs, resolvedPathStr, + remainingPath, true); + } catch (IOException ex) { + LOGGER.error(String.format( + "Got Exception while build resolve result." + + " ResultKind:%s, resolvedPathStr:%s," + + " targetOfResolvedPathStr:%s, remainingPath:%s," + + " will return null.", + resultKind, resolvedPathStr, targetOfResolvedPathStr, remainingPath), + ex); + return null; + } catch (URISyntaxException uex) { + LOGGER.error(String.format( + "Got Exception while build resolve result." + + " ResultKind:%s, resolvedPathStr:%s," + + " targetOfResolvedPathStr:%s, remainingPath:%s," + + " will return null.", + resultKind, resolvedPathStr, targetOfResolvedPathStr, remainingPath), + uex); + return null; + } } List> getMountPoints() { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPoint.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPoint.java new file mode 100644 index 0000000000000..aace7a2dba57d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPoint.java @@ -0,0 +1,289 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.viewfs; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.fs.viewfs.InodeTree.SlashPath; + +/** + * Regex mount point is build to implement regex based mount point. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +class RegexMountPoint { + private static final Logger LOGGER = + LoggerFactory.getLogger(RegexMountPoint.class.getName()); + + private InodeTree inodeTree; + private String srcPathRegex; + private Pattern srcPattern; + private String dstPath; + private String interceptorSettingsString; + private List interceptorList; + + public static final String SETTING_SRCREGEX_SEP = "#."; + public static final char INTERCEPTOR_SEP = ';'; + public static final char INTERCEPTOR_INTERNAL_SEP = ':'; + // ${var},$var + public static final Pattern VAR_PATTERN_IN_DEST = + Pattern.compile("\\$((\\{\\w+\\})|(\\w+))"); + + // Same var might have different representations. + // e.g. + // key => $key or key = > ${key} + private Map> varInDestPathMap; + + public Map> getVarInDestPathMap() { + return varInDestPathMap; + } + + RegexMountPoint(InodeTree inodeTree, String sourcePathRegex, + String destPath, String settingsStr) { + this.inodeTree = inodeTree; + this.srcPathRegex = sourcePathRegex; + this.dstPath = destPath; + this.interceptorSettingsString = settingsStr; + this.interceptorList = new ArrayList<>(); + } + + /** + * Initialize regex mount point. + * + * @throws IOException + */ + public void initialize() throws IOException { + try { + srcPattern = Pattern.compile(srcPathRegex); + } catch (PatternSyntaxException ex) { + throw new IOException( + "Failed to initialized mount point due to bad src path regex:" + + srcPathRegex + ", dstPath:" + dstPath, ex); + } + varInDestPathMap = getVarListInString(dstPath); + initializeInterceptors(); + } + + private void initializeInterceptors() throws IOException { + if (interceptorSettingsString == null + || interceptorSettingsString.isEmpty()) { + return; + } + String[] interceptorStrArray = + StringUtils.split(interceptorSettingsString, INTERCEPTOR_SEP); + for (String interceptorStr : interceptorStrArray) { + RegexMountPointInterceptor interceptor = + RegexMountPointInterceptorFactory.create(interceptorStr); + if (interceptor == null) { + throw new IOException( + "Illegal settings String " + interceptorSettingsString); + } + interceptor.initialize(); + interceptorList.add(interceptor); + } + } + + /** + * Get $var1 and $var2 style variables in string. + * + * @param input - the string to be process. + * @return + */ + public static Map> getVarListInString(String input) { + Map> varMap = new HashMap<>(); + Matcher matcher = VAR_PATTERN_IN_DEST.matcher(input); + while (matcher.find()) { + // $var or ${var} + String varName = matcher.group(0); + // var or {var} + String strippedVarName = matcher.group(1); + if (strippedVarName.startsWith("{")) { + // {varName} = > varName + strippedVarName = + strippedVarName.substring(1, strippedVarName.length() - 1); + } + varMap.putIfAbsent(strippedVarName, new HashSet<>()); + varMap.get(strippedVarName).add(varName); + } + return varMap; + } + + public String getSrcPathRegex() { + return srcPathRegex; + } + + public Pattern getSrcPattern() { + return srcPattern; + } + + public String getDstPath() { + return dstPath; + } + + public static Pattern getVarPatternInDest() { + return VAR_PATTERN_IN_DEST; + } + + /** + * Get resolved path from regex mount points. + * E.g. link: ^/user/(?\\w+) => s3://$user.apache.com/_${user} + * srcPath: is /user/hadoop/dir1 + * resolveLastComponent: true + * then return value is s3://hadoop.apache.com/_hadoop + * @param srcPath - the src path to resolve + * @param resolveLastComponent - whether resolve the path after last `/` + * @return mapped path of the mount point. + */ + public InodeTree.ResolveResult resolve(final String srcPath, + final boolean resolveLastComponent) { + String pathStrToResolve = getPathToResolve(srcPath, resolveLastComponent); + for (RegexMountPointInterceptor interceptor : interceptorList) { + pathStrToResolve = interceptor.interceptSource(pathStrToResolve); + } + LOGGER.debug("Path to resolve:" + pathStrToResolve + ", srcPattern:" + + getSrcPathRegex()); + Matcher srcMatcher = getSrcPattern().matcher(pathStrToResolve); + String parsedDestPath = getDstPath(); + int mappedCount = 0; + String resolvedPathStr = ""; + while (srcMatcher.find()) { + resolvedPathStr = pathStrToResolve.substring(0, srcMatcher.end()); + Map> varMap = getVarInDestPathMap(); + for (Map.Entry> entry : varMap.entrySet()) { + String regexGroupNameOrIndexStr = entry.getKey(); + Set groupRepresentationStrSetInDest = entry.getValue(); + parsedDestPath = replaceRegexCaptureGroupInPath( + parsedDestPath, srcMatcher, + regexGroupNameOrIndexStr, groupRepresentationStrSetInDest); + } + ++mappedCount; + } + if (0 == mappedCount) { + return null; + } + Path remainingPath = getRemainingPathStr(srcPath, resolvedPathStr); + for (RegexMountPointInterceptor interceptor : interceptorList) { + parsedDestPath = interceptor.interceptResolvedDestPathStr(parsedDestPath); + remainingPath = + interceptor.interceptRemainingPath(remainingPath); + } + InodeTree.ResolveResult resolveResult = inodeTree + .buildResolveResultForRegexMountPoint(InodeTree.ResultKind.EXTERNAL_DIR, + resolvedPathStr, parsedDestPath, remainingPath); + return resolveResult; + } + + private Path getRemainingPathStr( + String srcPath, + String resolvedPathStr) { + String remainingPathStr = srcPath.substring(resolvedPathStr.length()); + if (!remainingPathStr.startsWith("/")) { + remainingPathStr = "/" + remainingPathStr; + } + return new Path(remainingPathStr); + } + + private String getPathToResolve( + String srcPath, boolean resolveLastComponent) { + if (resolveLastComponent) { + return srcPath; + } + int lastSlashIndex = srcPath.lastIndexOf(SlashPath.toString()); + if (lastSlashIndex == -1) { + return null; + } + return srcPath.substring(0, lastSlashIndex); + } + + /** + * Use capture group named regexGroupNameOrIndexStr in mather to replace + * parsedDestPath. + * E.g. link: ^/user/(?\\w+) => s3://$user.apache.com/_${user} + * srcMatcher is from /user/hadoop. + * Then the params will be like following. + * parsedDestPath: s3://$user.apache.com/_${user}, + * regexGroupNameOrIndexStr: user + * groupRepresentationStrSetInDest: {user:$user; user:${user}} + * return value will be s3://hadoop.apache.com/_hadoop + * @param parsedDestPath + * @param srcMatcher + * @param regexGroupNameOrIndexStr + * @param groupRepresentationStrSetInDest + * @return return parsedDestPath while ${var},$var replaced or + * parsedDestPath nothing found. + */ + private String replaceRegexCaptureGroupInPath( + String parsedDestPath, + Matcher srcMatcher, + String regexGroupNameOrIndexStr, + Set groupRepresentationStrSetInDest) { + String groupValue = getRegexGroupValueFromMather( + srcMatcher, regexGroupNameOrIndexStr); + if (groupValue == null) { + return parsedDestPath; + } + for (String varName : groupRepresentationStrSetInDest) { + parsedDestPath = parsedDestPath.replace(varName, groupValue); + LOGGER.debug("parsedDestPath value is:" + parsedDestPath); + } + return parsedDestPath; + } + + /** + * Get matched capture group value from regex matched string. E.g. + * Regex: ^/user/(?\\w+), regexGroupNameOrIndexStr: userName + * then /user/hadoop should return hadoop while call + * getRegexGroupValueFromMather(matcher, usersName) + * or getRegexGroupValueFromMather(matcher, 1) + * + * @param srcMatcher - the matcher to be use + * @param regexGroupNameOrIndexStr - the regex group name or index + * @return - Null if no matched group named regexGroupNameOrIndexStr found. + */ + private String getRegexGroupValueFromMather( + Matcher srcMatcher, String regexGroupNameOrIndexStr) { + if (regexGroupNameOrIndexStr.matches("\\d+")) { + // group index + int groupIndex = Integer.parseUnsignedInt(regexGroupNameOrIndexStr); + if (groupIndex >= 0 && groupIndex <= srcMatcher.groupCount()) { + return srcMatcher.group(groupIndex); + } + } else { + // named group in regex + return srcMatcher.group(regexGroupNameOrIndexStr); + } + return null; + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPointInterceptor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPointInterceptor.java new file mode 100644 index 0000000000000..37f44b0a51579 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPointInterceptor.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.viewfs; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.Path; + +/** + * RegexMountPointInterceptor is a mechanism provided to intercept + * src and resolved path before/after resolution. + */ +@InterfaceAudience.LimitedPrivate("Common") +@InterfaceStability.Unstable +interface RegexMountPointInterceptor { + + /** + * Initialize interceptor and throws IOException if needed. + * @throws IOException + */ + void initialize() throws IOException; + + /** + * Intercept source before resolution. + * @param source + * @return + */ + String interceptSource(String source); + + /** + * Intercept parsed dest path and return a new one. + * @return intercepted string + */ + String interceptResolvedDestPathStr(String parsedDestPathStr); + + /** + * Intercept remaining path. + * @return intercepted string + */ + Path interceptRemainingPath(Path remainingPath); + + /** + * Get interceptor type. + * @return + */ + RegexMountPointInterceptorType getType(); + + /** + * Serialize the interceptor to a string. + * @return + */ + String serializeToString(); +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPointInterceptorFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPointInterceptorFactory.java new file mode 100644 index 0000000000000..fb564aa3a6e4d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPointInterceptorFactory.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.viewfs; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * The interceptor factory used to create RegexMountPoint interceptors. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +final class RegexMountPointInterceptorFactory { + + private RegexMountPointInterceptorFactory() { + + } + + /** + * interceptorSettingsString string should be like ${type}:${string}, + * e.g. replaceresolveddstpath:word1,word2. + * + * @param interceptorSettingsString + * @return Return interceptor based on setting or null on bad/unknown config. + */ + public static RegexMountPointInterceptor create( + String interceptorSettingsString) { + int typeTagIndex = interceptorSettingsString + .indexOf(RegexMountPoint.INTERCEPTOR_INTERNAL_SEP); + if (typeTagIndex == -1 || (typeTagIndex == ( + interceptorSettingsString.length() - 1))) { + return null; + } + String typeTag = interceptorSettingsString.substring(0, typeTagIndex).trim() + .toLowerCase(); + RegexMountPointInterceptorType interceptorType = + RegexMountPointInterceptorType.get(typeTag); + if (interceptorType == null) { + return null; + } + switch (interceptorType) { + case REPLACE_RESOLVED_DST_PATH: + RegexMountPointInterceptor interceptor = + RegexMountPointResolvedDstPathReplaceInterceptor + .deserializeFromString(interceptorSettingsString); + return interceptor; + default: + // impossible now + return null; + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPointInterceptorType.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPointInterceptorType.java new file mode 100644 index 0000000000000..ad953eba24ad9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPointInterceptorType.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.viewfs; + +import java.util.HashMap; +import java.util.Map; + +/** + * RegexMountPointInterceptorType. + */ +public enum RegexMountPointInterceptorType { + REPLACE_RESOLVED_DST_PATH("replaceresolveddstpath"); + + private final String configName; + private static final Map + INTERCEPTOR_TYPE_MAP + = new HashMap(); + + static { + for (RegexMountPointInterceptorType interceptorType + : RegexMountPointInterceptorType.values()) { + INTERCEPTOR_TYPE_MAP.put( + interceptorType.getConfigName(), interceptorType); + } + } + + RegexMountPointInterceptorType(String configName) { + this.configName = configName; + } + + public String getConfigName() { + return configName; + } + + public static RegexMountPointInterceptorType get(String configName) { + return INTERCEPTOR_TYPE_MAP.get(configName); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPointResolvedDstPathReplaceInterceptor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPointResolvedDstPathReplaceInterceptor.java new file mode 100644 index 0000000000000..18490dc57c5ac --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/RegexMountPointResolvedDstPathReplaceInterceptor.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.viewfs; + +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.Path; + +import static org.apache.hadoop.fs.viewfs.RegexMountPointInterceptorType.REPLACE_RESOLVED_DST_PATH; + +/** + * Implementation of RegexMountPointResolvedDstPathReplaceInterceptor. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +class RegexMountPointResolvedDstPathReplaceInterceptor + implements RegexMountPointInterceptor { + + private String srcRegexString; + private String replaceString; + private Pattern srcRegexPattern; + + RegexMountPointResolvedDstPathReplaceInterceptor(String srcRegex, + String replaceString) { + this.srcRegexString = srcRegex; + this.replaceString = replaceString; + this.srcRegexPattern = null; + } + + public String getSrcRegexString() { + return srcRegexString; + } + + public String getReplaceString() { + return replaceString; + } + + public Pattern getSrcRegexPattern() { + return srcRegexPattern; + } + + @Override + public void initialize() throws IOException { + try { + srcRegexPattern = Pattern.compile(srcRegexString); + } catch (PatternSyntaxException ex) { + throw new IOException( + "Initialize interceptor failed, srcRegx:" + srcRegexString, ex); + } + } + + /** + * Source won't be changed in the interceptor. + * + * @return source param string passed in. + */ + @Override + public String interceptSource(String source) { + return source; + } + + /** + * Intercept resolved path, e.g. + * Mount point /^(\\w+)/, ${1}.hadoop.net + * If incoming path is /user1/home/tmp/job1, + * then the resolved path str will be user1. + * + * @return intercepted string + */ + @Override + public String interceptResolvedDestPathStr( + String parsedDestPathStr) { + Matcher matcher = srcRegexPattern.matcher(parsedDestPathStr); + return matcher.replaceAll(replaceString); + } + + /** + * The interceptRemainingPath will just return the remainingPath passed in. + * + */ + @Override + public Path interceptRemainingPath(Path remainingPath) { + return remainingPath; + } + + @Override + public RegexMountPointInterceptorType getType() { + return REPLACE_RESOLVED_DST_PATH; + } + + @Override + public String serializeToString() { + return REPLACE_RESOLVED_DST_PATH.getConfigName() + + RegexMountPoint.INTERCEPTOR_INTERNAL_SEP + srcRegexString + + RegexMountPoint.INTERCEPTOR_INTERNAL_SEP + replaceString; + } + + /** + * Create interceptor from config string. The string should be in + * replaceresolvedpath:wordToReplace:replaceString + * Note that we'll assume there's no ':' in the regex for the moment. + * + * @return Interceptor instance or null on bad config. + */ + public static RegexMountPointResolvedDstPathReplaceInterceptor + deserializeFromString(String serializedString) { + String[] strings = serializedString + .split(Character.toString(RegexMountPoint.INTERCEPTOR_INTERNAL_SEP)); + // We'll assume there's no ':' in the regex for the moment. + if (strings.length != 3) { + return null; + } + //The format should be like replaceresolvedpath:wordToReplace:replaceString + return new RegexMountPointResolvedDstPathReplaceInterceptor(strings[1], + strings[2]); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 1fc531e05635d..473b51489a483 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -33,7 +33,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -42,8 +41,9 @@ import java.util.Map.Entry; import java.util.Objects; import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -114,6 +114,7 @@ protected FsGetter fsGetter() { static class InnerCache { private Map map = new HashMap<>(); private FsGetter fsCreator; + private ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(); InnerCache(FsGetter fsCreator) { this.fsCreator = fsCreator; @@ -121,12 +122,27 @@ static class InnerCache { FileSystem get(URI uri, Configuration config) throws IOException { Key key = new Key(uri); - if (map.get(key) == null) { - FileSystem fs = fsCreator.getNewInstance(uri, config); + FileSystem fs = null; + try { + rwLock.readLock().lock(); + fs = map.get(key); + if (fs != null) { + return fs; + } + } finally { + rwLock.readLock().unlock(); + } + try { + rwLock.writeLock().lock(); + fs = map.get(key); + if (fs != null) { + return fs; + } + fs = fsCreator.getNewInstance(uri, config); map.put(key, fs); return fs; - } else { - return map.get(key); + } finally { + rwLock.writeLock().unlock(); } } @@ -140,9 +156,13 @@ void closeAll() { } } - InnerCache unmodifiableCache() { - map = Collections.unmodifiableMap(map); - return this; + void clear() { + try { + rwLock.writeLock().lock(); + map.clear(); + } finally { + rwLock.writeLock().unlock(); + } } /** @@ -259,13 +279,14 @@ public String getScheme() { } /** - * Returns the ViewFileSystem type. - * @return viewfs + * Returns false as it does not support to add fallback link automatically on + * no mounts. */ - String getType() { - return FsConstants.VIEWFS_TYPE; + boolean supportAutoAddingFallbackOnNoMounts() { + return false; } + /** * Called after a new FileSystem instance is constructed. * @param theUri a uri whose authority section names the host, port, etc. for @@ -293,19 +314,19 @@ public void initialize(final URI theUri, final Configuration conf) try { myUri = new URI(getScheme(), authority, "/", null, null); boolean initingUriAsFallbackOnNoMounts = - !FsConstants.VIEWFS_TYPE.equals(getType()); - fsState = new InodeTree(conf, tableName, theUri, + supportAutoAddingFallbackOnNoMounts(); + fsState = new InodeTree(conf, tableName, myUri, initingUriAsFallbackOnNoMounts) { @Override protected FileSystem getTargetFileSystem(final URI uri) throws URISyntaxException, IOException { - FileSystem fs; - if (enableInnerCache) { - fs = innerCache.get(uri, config); - } else { - fs = fsGetter.get(uri, config); - } - return new ChRootedFileSystem(fs, uri); + FileSystem fs; + if (enableInnerCache) { + fs = innerCache.get(uri, config); + } else { + fs = fsGetter.get(uri, config); + } + return new ChRootedFileSystem(fs, uri); } @Override @@ -334,7 +355,7 @@ protected FileSystem getTargetFileSystem(final String settings, // All fs instances are created and cached on startup. The cache is // readonly after the initialize() so the concurrent access of the cache // is safe. - cache = innerCache.unmodifiableCache(); + cache = innerCache; } } @@ -406,7 +427,7 @@ public FSDataOutputStream append(final Path f, final int bufferSize, fsState.resolve(getUriPath(f), true); return res.targetFileSystem.append(res.remainingPath, bufferSize, progress); } - + @Override public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, EnumSet flags, int bufferSize, short replication, @@ -649,18 +670,52 @@ public FSDataInputStream open(final Path f, final int bufferSize) @Override public boolean rename(final Path src, final Path dst) throws IOException { // passing resolveLastComponet as false to catch renaming a mount point to - // itself. We need to catch this as an internal operation and fail. - InodeTree.ResolveResult resSrc = - fsState.resolve(getUriPath(src), false); - + // itself. We need to catch this as an internal operation and fail if no + // fallback. + InodeTree.ResolveResult resSrc = + fsState.resolve(getUriPath(src), false); + if (resSrc.isInternalDir()) { - throw readOnlyMountTable("rename", src); + if (fsState.getRootFallbackLink() == null) { + // If fallback is null, we can't rename from src. + throw readOnlyMountTable("rename", src); + } + InodeTree.ResolveResult resSrcWithLastComp = + fsState.resolve(getUriPath(src), true); + if (resSrcWithLastComp.isInternalDir() || resSrcWithLastComp + .isLastInternalDirLink()) { + throw readOnlyMountTable("rename", src); + } else { + // This is fallback and let's set the src fs with this fallback + resSrc = resSrcWithLastComp; + } } - - InodeTree.ResolveResult resDst = - fsState.resolve(getUriPath(dst), false); + + InodeTree.ResolveResult resDst = + fsState.resolve(getUriPath(dst), false); + if (resDst.isInternalDir()) { - throw readOnlyMountTable("rename", dst); + if (fsState.getRootFallbackLink() == null) { + // If fallback is null, we can't rename to dst. + throw readOnlyMountTable("rename", dst); + } + // if the fallback exist, we may have chance to rename to fallback path + // where dst parent is matching to internalDir. + InodeTree.ResolveResult resDstWithLastComp = + fsState.resolve(getUriPath(dst), true); + if (resDstWithLastComp.isInternalDir()) { + // We need to get fallback here. If matching fallback path not exist, it + // will fail later. This is a very special case: Even though we are on + // internal directory, we should allow to rename, so that src files will + // moved under matching fallback dir. + resDst = new InodeTree.ResolveResult( + InodeTree.ResultKind.INTERNAL_DIR, + fsState.getRootFallbackLink().getTargetFileSystem(), "/", + new Path(resDstWithLastComp.resolvedPath), false); + } else { + // The link resolved to some target fs or fallback fs. + resDst = resDstWithLastComp; + } } URI srcUri = resSrc.targetFileSystem.getUri(); @@ -938,6 +993,12 @@ public FileSystem[] getChildFileSystems() { FileSystem targetFs = mountPoint.target.targetFileSystem; children.addAll(Arrays.asList(targetFs.getChildFileSystems())); } + + if (fsState.isRootInternalDir() && fsState.getRootFallbackLink() != null) { + children.addAll(Arrays.asList( + fsState.getRootFallbackLink().targetFileSystem + .getChildFileSystems())); + } return children.toArray(new FileSystem[]{}); } @@ -1256,6 +1317,23 @@ public boolean delete(final Path f) public BlockLocation[] getFileBlockLocations(final FileStatus fs, final long start, final long len) throws FileNotFoundException, IOException { + + // When application calls listFiles on internalDir, it would return + // RemoteIterator from InternalDirOfViewFs. If there is a fallBack, there + // is a chance of files exists under that internalDir in fallback. + // Iterator#next will call getFileBlockLocations with that files. So, we + // should return getFileBlockLocations on fallback. See HDFS-15532. + if (!InodeTree.SlashPath.equals(fs.getPath()) && this.fsState + .getRootFallbackLink() != null) { + FileSystem linkedFallbackFs = + this.fsState.getRootFallbackLink().getTargetFileSystem(); + Path parent = Path.getPathWithoutSchemeAndAuthority( + new Path(theInternalDir.fullPath)); + Path pathToFallbackFs = new Path(parent, fs.getPath().getName()); + return linkedFallbackFs + .getFileBlockLocations(pathToFallbackFs, start, len); + } + checkPathIsSlash(fs.getPath()); throw new FileNotFoundException("Path points to dir not a file"); } @@ -1421,7 +1499,7 @@ public FsStatus getStatus(Path p) throws IOException { @Override public boolean mkdirs(Path dir, FsPermission permission) - throws AccessControlException, FileAlreadyExistsException { + throws IOException { if (theInternalDir.isRoot() && dir == null) { throw new FileAlreadyExistsException("/ already exits"); } @@ -1451,7 +1529,7 @@ public boolean mkdirs(Path dir, FsPermission permission) .append(linkedFallbackFs.getUri()); LOG.debug(msg.toString(), e); } - return false; + throw e; } } @@ -1459,8 +1537,7 @@ public boolean mkdirs(Path dir, FsPermission permission) } @Override - public boolean mkdirs(Path dir) - throws AccessControlException, FileAlreadyExistsException { + public boolean mkdirs(Path dir) throws IOException { return mkdirs(dir, null); } @@ -1684,6 +1761,7 @@ public void close() throws IOException { super.close(); if (enableInnerCache && cache != null) { cache.closeAll(); + cache.clear(); } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java index 2165a3f9ee688..12877ccf4bb15 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java @@ -33,77 +33,90 @@ import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME; -/****************************************************************************** - * This class is extended from the ViewFileSystem for the overloaded scheme - * file system. Mount link configurations and in-memory mount table - * building behaviors are inherited from ViewFileSystem. Unlike ViewFileSystem - * scheme (viewfs://), the users would be able to use any scheme. +/** + *

This class is extended from the ViewFileSystem for the overloaded + * scheme file system. Mount link configurations and in-memory mount table + * building behaviors are inherited from ViewFileSystem. Unlike + * ViewFileSystem scheme (viewfs://), the users would be able to use + * any scheme.

* - * To use this class, the following configurations need to be added in - * core-site.xml file. - * 1) fs..impl - * = org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme - * 2) fs.viewfs.overload.scheme.target..impl - * = " + *

To use this class, the following configurations need to be added in + * core-site.xml file.
+ * 1) fs.{@literal }.impl + * = org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme
+ * 2) fs.viewfs.overload.scheme.target.{@literal }.impl + * = {@literal >}

* - * Here can be any scheme, but with that scheme there should be a - * hadoop compatible file system available. Second configuration value should - * be the respective scheme's file system implementation class. + *

Here {@literal } can be any scheme, but with that scheme there + * should be a hadoop compatible file system available. Second configuration + * value should be the respective scheme's file system implementation class. * Example: if scheme is configured with "hdfs", then the 2nd configuration * class name will be org.apache.hadoop.hdfs.DistributedFileSystem. * if scheme is configured with "s3a", then the 2nd configuration class name - * will be org.apache.hadoop.fs.s3a.S3AFileSystem. + * will be org.apache.hadoop.fs.s3a.S3AFileSystem.

* - * Use Case 1: - * =========== + *

Use Case 1:
+ * ===========
* If users want some of their existing cluster (hdfs://Cluster) * data to mount with other hdfs and object store clusters(hdfs://NN1, - * o3fs://bucket1.volume1/, s3a://bucket1/) + * o3fs://bucket1.volume1/, s3a://bucket1/)

* - * fs.viewfs.mounttable.Cluster.link./user = hdfs://NN1/user - * fs.viewfs.mounttable.Cluster.link./data = o3fs://bucket1.volume1/data + *

+ * fs.viewfs.mounttable.Cluster.link./user = hdfs://NN1/user
+ * fs.viewfs.mounttable.Cluster.link./data = o3fs://bucket1.volume1/data
* fs.viewfs.mounttable.Cluster.link./backup = s3a://bucket1/backup/ + *

* + *

* Op1: Create file hdfs://Cluster/user/fileA will go to hdfs://NN1/user/fileA + *
* Op2: Create file hdfs://Cluster/data/datafile will go to - * o3fs://bucket1.volume1/data/datafile + * o3fs://bucket1.volume1/data/datafile
* Op3: Create file hdfs://Cluster/backup/data.zip will go to * s3a://bucket1/backup/data.zip + *

* - * Use Case 2: - * =========== + *

Use Case 2:
+ * ===========
* If users want some of their existing cluster (s3a://bucketA/) * data to mount with other hdfs and object store clusters - * (hdfs://NN1, o3fs://bucket1.volume1/) + * (hdfs://NN1, o3fs://bucket1.volume1/)

* - * fs.viewfs.mounttable.bucketA.link./user = hdfs://NN1/user - * fs.viewfs.mounttable.bucketA.link./data = o3fs://bucket1.volume1/data + *

+ * fs.viewfs.mounttable.bucketA.link./user = hdfs://NN1/user
+ * fs.viewfs.mounttable.bucketA.link./data = o3fs://bucket1.volume1/data
* fs.viewfs.mounttable.bucketA.link./salesDB = s3a://bucketA/salesDB/ + *

* + *

* Op1: Create file s3a://bucketA/user/fileA will go to hdfs://NN1/user/fileA + *
* Op2: Create file s3a://bucketA/data/datafile will go to - * o3fs://bucket1.volume1/data/datafile + * o3fs://bucket1.volume1/data/datafile
* Op3: Create file s3a://bucketA/salesDB/dbfile will go to * s3a://bucketA/salesDB/dbfile + *

* - * Note: + *

Note:
* (1) In ViewFileSystemOverloadScheme, by default the mount links will be * represented as non-symlinks. If you want to change this behavior, please see - * {@link ViewFileSystem#listStatus(Path)} + * {@link ViewFileSystem#listStatus(Path)}
* (2) In ViewFileSystemOverloadScheme, only the initialized uri's hostname will * be considered as the mount table name. When the passed uri has hostname:port, * it will simply ignore the port number and only hostname will be considered as - * the mount table name. + * the mount table name.
* (3) If there are no mount links configured with the initializing uri's * hostname as the mount table name, then it will automatically consider the - * current uri as fallback( ex: fs.viewfs.mounttable..linkFallBack) - * target fs uri. - *****************************************************************************/ + * current uri as fallback( ex: + * {@literal fs.viewfs.mounttable..linkFallback}) target fs uri. + *

+ */ @InterfaceAudience.LimitedPrivate({ "MapReduce", "HBase", "Hive" }) @InterfaceStability.Evolving public class ViewFileSystemOverloadScheme extends ViewFileSystem { private URI myUri; + private boolean supportAutoAddingFallbackOnNoMounts = true; public ViewFileSystemOverloadScheme() throws IOException { super(); } @@ -114,11 +127,19 @@ public String getScheme() { } /** - * Returns the ViewFileSystem type. - * @return viewfs + * By default returns false as ViewFileSystemOverloadScheme supports auto + * adding fallback on no mounts. */ - String getType() { - return FsConstants.VIEWFSOS_TYPE; + public boolean supportAutoAddingFallbackOnNoMounts() { + return this.supportAutoAddingFallbackOnNoMounts; + } + + /** + * Sets whether to add fallback automatically when no mount points found. + */ + public void setSupportAutoAddingFallbackOnNoMounts( + boolean addAutoFallbackOnNoMounts) { + this.supportAutoAddingFallbackOnNoMounts = addAutoFallbackOnNoMounts; } @Override @@ -155,12 +176,13 @@ public void initialize(URI theUri, Configuration conf) throws IOException { /** * This method is overridden because in ViewFileSystemOverloadScheme if * overloaded scheme matches with mounted target fs scheme, file system - * should be created without going into fs..impl based resolution. - * Otherwise it will end up in an infinite loop as the target will be - * resolved again to ViewFileSystemOverloadScheme as fs..impl points - * to ViewFileSystemOverloadScheme. So, below method will initialize the - * fs.viewfs.overload.scheme.target..impl. Other schemes can - * follow fs.newInstance + * should be created without going into {@literal fs..impl} based + * resolution. Otherwise it will end up in an infinite loop as the target + * will be resolved again to ViewFileSystemOverloadScheme as + * {@literal fs..impl} points to ViewFileSystemOverloadScheme. + * So, below method will initialize the + * {@literal fs.viewfs.overload.scheme.target..impl}. + * Other schemes can follow fs.newInstance */ @Override protected FsGetter fsGetter() { @@ -170,7 +192,7 @@ protected FsGetter fsGetter() { /** * This class checks whether the rooScheme is same as URI scheme. If both are * same, then it will initialize file systems by using the configured - * fs.viewfs.overload.scheme.target..impl class. + * {@literal fs.viewfs.overload.scheme.target..impl} class. */ static class ChildFsGetter extends FsGetter { @@ -287,4 +309,67 @@ public FileSystem getRawFileSystem(Path path, Configuration conf) } } + /** + * Gets the mount path info, which contains the target file system and + * remaining path to pass to the target file system. + */ + public MountPathInfo getMountPathInfo(Path path, + Configuration conf) throws IOException { + InodeTree.ResolveResult res; + try { + res = fsState.resolve(getUriPath(path), true); + FileSystem fs = res.isInternalDir() ? + (fsState.getRootFallbackLink() != null ? + ((ChRootedFileSystem) fsState + .getRootFallbackLink().targetFileSystem).getMyFs() : + fsGetter().get(path.toUri(), conf)) : + ((ChRootedFileSystem) res.targetFileSystem).getMyFs(); + return new MountPathInfo(res.remainingPath, res.resolvedPath, + fs); + } catch (FileNotFoundException e) { + // No link configured with passed path. + throw new NotInMountpointException(path, + "No link found for the given path."); + } + } + + /** + * A class to maintain the target file system and a path to pass to the target + * file system. + */ + public static class MountPathInfo { + private Path pathOnTarget; + private T targetFs; + + public MountPathInfo(Path pathOnTarget, String resolvedPath, T targetFs) { + this.pathOnTarget = pathOnTarget; + this.targetFs = targetFs; + } + + public Path getPathOnTarget() { + return this.pathOnTarget; + } + + public T getTargetFs() { + return this.targetFs; + } + } + + /** + * @return Gets the fallback file system configured. Usually, this will be the + * default cluster. + */ + public FileSystem getFallbackFileSystem() { + if (fsState.getRootFallbackLink() == null) { + return null; + } + return ((ChRootedFileSystem) fsState.getRootFallbackLink().targetFileSystem) + .getMyFs(); + } + + @Override + @InterfaceAudience.LimitedPrivate("HDFS") + public URI canonicalizeUri(URI uri) { + return super.canonicalizeUri(uri); + } } \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 95b596bde367d..a7d56fa56f443 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -34,7 +34,7 @@ import java.util.Set; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -548,23 +548,60 @@ public boolean truncate(final Path f, final long newLength) public void renameInternal(final Path src, final Path dst, final boolean overwrite) throws IOException, UnresolvedLinkException { // passing resolveLastComponet as false to catch renaming a mount point - // itself we need to catch this as an internal operation and fail. - InodeTree.ResolveResult resSrc = - fsState.resolve(getUriPath(src), false); - + // itself we need to catch this as an internal operation and fail if no + // fallback. + InodeTree.ResolveResult resSrc = + fsState.resolve(getUriPath(src), false); + if (resSrc.isInternalDir()) { - throw new AccessControlException( - "Cannot Rename within internal dirs of mount table: src=" + src - + " is readOnly"); + if (fsState.getRootFallbackLink() == null) { + // If fallback is null, we can't rename from src. + throw new AccessControlException( + "Cannot Rename within internal dirs of mount table: src=" + src + + " is readOnly"); + } + InodeTree.ResolveResult resSrcWithLastComp = + fsState.resolve(getUriPath(src), true); + if (resSrcWithLastComp.isInternalDir() || resSrcWithLastComp + .isLastInternalDirLink()) { + throw new AccessControlException( + "Cannot Rename within internal dirs of mount table: src=" + src + + " is readOnly"); + } else { + // This is fallback and let's set the src fs with this fallback + resSrc = resSrcWithLastComp; + } } InodeTree.ResolveResult resDst = - fsState.resolve(getUriPath(dst), false); + fsState.resolve(getUriPath(dst), false); + if (resDst.isInternalDir()) { - throw new AccessControlException( - "Cannot Rename within internal dirs of mount table: dest=" + dst - + " is readOnly"); + if (fsState.getRootFallbackLink() == null) { + // If fallback is null, we can't rename to dst. + throw new AccessControlException( + "Cannot Rename within internal dirs of mount table: dest=" + dst + + " is readOnly"); + } + // if the fallback exist, we may have chance to rename to fallback path + // where dst parent is matching to internalDir. + InodeTree.ResolveResult resDstWithLastComp = + fsState.resolve(getUriPath(dst), true); + if (resDstWithLastComp.isInternalDir()) { + // We need to get fallback here. If matching fallback path not exist, it + // will fail later. This is a very special case: Even though we are on + // internal directory, we should allow to rename, so that src files will + // moved under matching fallback dir. + resDst = new InodeTree.ResolveResult( + InodeTree.ResultKind.INTERNAL_DIR, + fsState.getRootFallbackLink().getTargetFileSystem(), "/", + new Path(resDstWithLastComp.resolvedPath), false); + } else { + // The link resolved to some target fs or fallback fs. + resDst = resDstWithLastComp; + } } + //Alternate 1: renames within same file system URI srcUri = resSrc.targetFileSystem.getUri(); URI dstUri = resDst.targetFileSystem.getUri(); @@ -981,6 +1018,21 @@ public boolean delete(final Path f, final boolean recursive) @Override public BlockLocation[] getFileBlockLocations(final Path f, final long start, final long len) throws FileNotFoundException, IOException { + // When application calls listFiles on internalDir, it would return + // RemoteIterator from InternalDirOfViewFs. If there is a fallBack, there + // is a chance of files exists under that internalDir in fallback. + // Iterator#next will call getFileBlockLocations with that files. So, we + // should return getFileBlockLocations on fallback. See HDFS-15532. + if (!InodeTree.SlashPath.equals(f) && this.fsState + .getRootFallbackLink() != null) { + AbstractFileSystem linkedFallbackFs = + this.fsState.getRootFallbackLink().getTargetFileSystem(); + Path parent = Path.getPathWithoutSchemeAndAuthority( + new Path(theInternalDir.fullPath)); + Path pathToFallbackFs = new Path(parent, f.getName()); + return linkedFallbackFs + .getFileBlockLocations(pathToFallbackFs, start, len); + } checkPathIsSlash(f); throw new FileNotFoundException("Path points to dir not a file"); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java index 828a17bcb972e..08f6df5a78661 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java @@ -43,8 +43,8 @@ import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.KeeperException.Code; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FailoverController.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FailoverController.java index 5ad71f373f2d0..e7ed7304988cb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FailoverController.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FailoverController.java @@ -28,7 +28,7 @@ import org.apache.hadoop.ha.HAServiceProtocol.RequestSource; import org.apache.hadoop.ipc.RPC; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java index 34e37650ade1c..c6949e561e2a2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java @@ -39,7 +39,7 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceTarget.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceTarget.java index ff9658f1bbc03..2e6b1fe113479 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceTarget.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceTarget.java @@ -31,7 +31,7 @@ import org.apache.hadoop.ha.protocolPB.ZKFCProtocolClientSideTranslatorPB; import org.apache.hadoop.net.NetUtils; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; /** * Represents a target of the client side HA administration commands. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java index 16c30752edc20..732058649bc50 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java @@ -32,7 +32,7 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.util.Daemon; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java index b0cead56ac0e7..7f4a0790a3bc1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java @@ -27,8 +27,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ReflectionUtils; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java index 6363063abf2e2..3ae8394b62342 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java @@ -24,7 +24,7 @@ import org.apache.hadoop.conf.Configured; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.util.Shell; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java index e0c2f4d9e7b77..a13b592e5cb15 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java @@ -25,7 +25,7 @@ import org.apache.hadoop.conf.Configured; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import com.jcraft.jsch.ChannelExec; import com.jcraft.jsch.JSch; import com.jcraft.jsch.JSchException; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java index 3718b7cdb0cc0..16d7bf7ba4b5d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java @@ -50,9 +50,9 @@ import org.apache.hadoop.util.ToolRunner; import org.apache.zookeeper.data.ACL; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index 8b69d57e8120e..dfc9436a6f03f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -50,9 +50,9 @@ import javax.servlet.http.HttpServletRequestWrapper; import javax.servlet.http.HttpServletResponse; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import com.sun.jersey.spi.container.servlet.ServletContainer; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java index 44e0bdce5edd6..8f8c55d5e4654 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java @@ -29,7 +29,7 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Abstract base class for MapWritable and SortedMapWritable diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputBuffer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputBuffer.java index 4c2fa67f8f24c..e21dc2f632317 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputBuffer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputBuffer.java @@ -23,7 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** A reusable {@link DataOutput} implementation that writes to an in-memory * buffer. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java index bbedf2a2dc371..566fd864568c5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.io; -import com.google.common.collect.ComparisonChain; +import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; import org.apache.commons.lang3.builder.HashCodeBuilder; import java.nio.ByteBuffer; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java index 5af6602b87886..1ef2119b688fd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java @@ -26,7 +26,7 @@ import org.slf4j.LoggerFactory; import sun.misc.Unsafe; -import com.google.common.primitives.UnsignedBytes; +import org.apache.hadoop.thirdparty.com.google.common.primitives.UnsignedBytes; /** * Utility code to do optimized byte-array comparison. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java index 804d365450692..b5acc35a4ce6d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java @@ -29,8 +29,9 @@ import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_WILLNEED; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,7 +60,17 @@ public static ReadaheadPool getInstance() { return instance; } } - + + @VisibleForTesting + public static void resetInstance() { + synchronized (ReadaheadPool.class) { + if (instance != null) { + instance.pool.shutdownNow(); + instance = null; + } + } + } + private ReadaheadPool() { pool = new ThreadPoolExecutor(POOL_SIZE, MAX_POOL_SIZE, 3L, TimeUnit.SECONDS, new ArrayBlockingQueue(CAPACITY)); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java index f14d99227c7cc..e6bd81a0bc8d6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java @@ -32,7 +32,7 @@ import org.apache.hadoop.io.nativeio.NativeIO.POSIX.Stat; import org.apache.hadoop.security.UserGroupInformation; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * This class provides secure APIs for opening and creating files on the local diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java index fec0a4ac81f2f..3f4649f04dc9a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java @@ -24,7 +24,7 @@ import java.rmi.server.UID; import java.security.MessageDigest; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.util.Options; import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.Options.CreateOpts; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java index 716de3deb4278..6022b99544114 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java @@ -77,6 +77,7 @@ protected CharsetDecoder initialValue() { private byte[] bytes = EMPTY_BYTES; private int length = 0; + private int textLength = -1; /** * Construct an empty text string. @@ -131,6 +132,17 @@ public int getLength() { return length; } + /** + * Returns the length of this text. The length is equal to the number of + * Unicode code units in the text. + */ + public int getTextLength() { + if (textLength < 0) { + textLength = toString().length(); + } + return textLength; + } + /** * Returns the Unicode Scalar Value (32-bit integer value) * for the character at position. Note that this @@ -204,6 +216,7 @@ public void set(String string) { ByteBuffer bb = encode(string, true); bytes = bb.array(); length = bb.limit(); + textLength = string.length(); } catch (CharacterCodingException e) { throw new RuntimeException("Should not have happened", e); } @@ -221,6 +234,7 @@ public void set(byte[] utf8) { */ public void set(Text other) { set(other.getBytes(), 0, other.getLength()); + this.textLength = other.textLength; } /** @@ -234,6 +248,7 @@ public void set(byte[] utf8, int start, int len) { ensureCapacity(len); System.arraycopy(utf8, start, bytes, 0, len); this.length = len; + this.textLength = -1; } /** @@ -251,6 +266,7 @@ public void append(byte[] utf8, int start, int len) { } System.arraycopy(utf8, start, bytes, length, len); length += len; + textLength = -1; } /** @@ -263,6 +279,7 @@ public void append(byte[] utf8, int start, int len) { */ public void clear() { length = 0; + textLength = -1; } /** @@ -327,6 +344,7 @@ public void readWithKnownLength(DataInput in, int len) throws IOException { ensureCapacity(len); in.readFully(bytes, 0, len); length = len; + textLength = -1; } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java index f103aad4e8596..2ac2ca65173f0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java @@ -28,9 +28,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ReflectionUtils; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; +import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Decompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Decompressor.java index 3808003de291d..e9558fab87325 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Decompressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Decompressor.java @@ -92,7 +92,7 @@ public interface Decompressor { * {@link #needsInput()} should be called in order to determine if more * input data is required. * - * @param b Buffer for the compressed data + * @param b Buffer for the uncompressed data * @param off Start offset of the data * @param len Size of the buffer * @return The actual number of bytes of uncompressed data. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java index 756ccf3c8ed1f..d66b6f0d0237d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java @@ -22,7 +22,7 @@ import java.io.IOException; import java.io.InputStream; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java index 686f30c9f89a2..77cf36a339b34 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java @@ -28,7 +28,6 @@ import org.apache.hadoop.io.compress.snappy.SnappyDecompressor; import org.apache.hadoop.io.compress.snappy.SnappyDecompressor.SnappyDirectDecompressor; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.util.NativeCodeLoader; /** * This class creates snappy compressors/decompressors. @@ -56,37 +55,6 @@ public Configuration getConf() { return conf; } - /** - * Are the native snappy libraries loaded & initialized? - */ - public static void checkNativeCodeLoaded() { - if (!NativeCodeLoader.buildSupportsSnappy()) { - throw new RuntimeException("native snappy library not available: " + - "this version of libhadoop was built without " + - "snappy support."); - } - if (!NativeCodeLoader.isNativeCodeLoaded()) { - throw new RuntimeException("Failed to load libhadoop."); - } - if (!SnappyCompressor.isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available: " + - "SnappyCompressor has not been loaded."); - } - if (!SnappyDecompressor.isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available: " + - "SnappyDecompressor has not been loaded."); - } - } - - public static boolean isNativeCodeLoaded() { - return SnappyCompressor.isNativeCodeLoaded() && - SnappyDecompressor.isNativeCodeLoaded(); - } - - public static String getLibraryName() { - return SnappyCompressor.getLibraryName(); - } - /** * Create a {@link CompressionOutputStream} that will write to the given * {@link OutputStream}. @@ -115,7 +83,6 @@ public CompressionOutputStream createOutputStream(OutputStream out) public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException { - checkNativeCodeLoaded(); int bufferSize = conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); @@ -133,7 +100,6 @@ public CompressionOutputStream createOutputStream(OutputStream out, */ @Override public Class getCompressorType() { - checkNativeCodeLoaded(); return SnappyCompressor.class; } @@ -144,7 +110,6 @@ public Class getCompressorType() { */ @Override public Compressor createCompressor() { - checkNativeCodeLoaded(); int bufferSize = conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); @@ -179,7 +144,6 @@ public CompressionInputStream createInputStream(InputStream in) public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException { - checkNativeCodeLoaded(); return new BlockDecompressorStream(in, decompressor, conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT)); @@ -192,7 +156,6 @@ public CompressionInputStream createInputStream(InputStream in, */ @Override public Class getDecompressorType() { - checkNativeCodeLoaded(); return SnappyDecompressor.class; } @@ -203,7 +166,6 @@ public Class getDecompressorType() { */ @Override public Decompressor createDecompressor() { - checkNativeCodeLoaded(); int bufferSize = conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); @@ -215,7 +177,7 @@ public Decompressor createDecompressor() { */ @Override public DirectDecompressor createDirectDecompressor() { - return isNativeCodeLoaded() ? new SnappyDirectDecompressor() : null; + return new SnappyDirectDecompressor(); } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java index 3792c365b4d98..5789ff6ca1150 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java @@ -44,6 +44,7 @@ public class Lz4Compressor implements Compressor { private byte[] userBuf = null; private int userBufOff = 0, userBufLen = 0; private boolean finish, finished; + private int dstCapacity; private long bytesRead = 0L; private long bytesWritten = 0L; @@ -77,8 +78,14 @@ public Lz4Compressor(int directBufferSize, boolean useLz4HC) { this.directBufferSize = directBufferSize; uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); - compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); - compressedDirectBuf.position(directBufferSize); + + // Compression is guaranteed to succeed if 'dstCapacity' >= + // LZ4_compressBound(srcSize) + // whereas LZ4_compressBound(isize) is (isize) + ((isize)/255) + 16) + this.dstCapacity = (directBufferSize) + ((directBufferSize) / 255) + 16; + + compressedDirectBuf = ByteBuffer.allocateDirect(this.dstCapacity); + compressedDirectBuf.position(this.dstCapacity); } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java index 3d386800e4d87..2d514705d1e42 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java @@ -24,9 +24,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.compress.Compressor; -import org.apache.hadoop.util.NativeCodeLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.xerial.snappy.Snappy; /** * A {@link Compressor} based on the snappy compression algorithm. @@ -48,24 +48,6 @@ public class SnappyCompressor implements Compressor { private long bytesRead = 0L; private long bytesWritten = 0L; - private static boolean nativeSnappyLoaded = false; - - static { - if (NativeCodeLoader.isNativeCodeLoaded() && - NativeCodeLoader.buildSupportsSnappy()) { - try { - initIDs(); - nativeSnappyLoaded = true; - } catch (Throwable t) { - LOG.error("failed to load SnappyCompressor", t); - } - } - } - - public static boolean isNativeCodeLoaded() { - return nativeSnappyLoaded; - } - /** * Creates a new compressor. * @@ -225,7 +207,7 @@ public int compress(byte[] b, int off, int len) } // Compress data - n = compressBytesDirect(); + n = compressDirectBuf(); compressedDirectBuf.limit(n); uncompressedDirectBuf.clear(); // snappy consumes all buffer input @@ -291,9 +273,16 @@ public long getBytesWritten() { public void end() { } - private native static void initIDs(); - - private native int compressBytesDirect(); - - public native static String getLibraryName(); + private int compressDirectBuf() throws IOException { + if (uncompressedDirectBufLen == 0) { + return 0; + } else { + // Set the position and limit of `uncompressedDirectBuf` for reading + uncompressedDirectBuf.limit(uncompressedDirectBufLen).position(0); + int size = Snappy.compress((ByteBuffer) uncompressedDirectBuf, + (ByteBuffer) compressedDirectBuf); + uncompressedDirectBufLen = 0; + return size; + } + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java index f31b76c347c5c..d3775e286e895 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java @@ -24,9 +24,9 @@ import org.apache.hadoop.io.compress.Decompressor; import org.apache.hadoop.io.compress.DirectDecompressor; -import org.apache.hadoop.util.NativeCodeLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.xerial.snappy.Snappy; /** * A {@link Decompressor} based on the snappy compression algorithm. @@ -45,24 +45,6 @@ public class SnappyDecompressor implements Decompressor { private int userBufOff = 0, userBufLen = 0; private boolean finished; - private static boolean nativeSnappyLoaded = false; - - static { - if (NativeCodeLoader.isNativeCodeLoaded() && - NativeCodeLoader.buildSupportsSnappy()) { - try { - initIDs(); - nativeSnappyLoaded = true; - } catch (Throwable t) { - LOG.error("failed to load SnappyDecompressor", t); - } - } - } - - public static boolean isNativeCodeLoaded() { - return nativeSnappyLoaded; - } - /** * Creates a new compressor. * @@ -201,7 +183,7 @@ public boolean finished() { * {@link #needsInput()} should be called in order to determine if more * input data is required. * - * @param b Buffer for the compressed data + * @param b Buffer for the uncompressed data * @param off Start offset of the data * @param len Size of the buffer * @return The actual number of bytes of compressed data. @@ -232,7 +214,7 @@ public int decompress(byte[] b, int off, int len) uncompressedDirectBuf.limit(directBufferSize); // Decompress data - n = decompressBytesDirect(); + n = decompressDirectBuf(); uncompressedDirectBuf.limit(n); if (userBufLen <= 0) { @@ -276,10 +258,20 @@ public void end() { // do nothing } - private native static void initIDs(); + private int decompressDirectBuf() throws IOException { + if (compressedDirectBufLen == 0) { + return 0; + } else { + // Set the position and limit of `compressedDirectBuf` for reading + compressedDirectBuf.limit(compressedDirectBufLen).position(0); + int size = Snappy.uncompress((ByteBuffer) compressedDirectBuf, + (ByteBuffer) uncompressedDirectBuf); + compressedDirectBufLen = 0; + compressedDirectBuf.clear(); + return size; + } + } - private native int decompressBytesDirect(); - int decompressDirect(ByteBuffer src, ByteBuffer dst) throws IOException { assert (this instanceof SnappyDirectDecompressor); @@ -298,7 +290,7 @@ int decompressDirect(ByteBuffer src, ByteBuffer dst) throws IOException { directBufferSize = dst.remaining(); int n = 0; try { - n = decompressBytesDirect(); + n = decompressDirectBuf(); presliced.position(presliced.position() + n); // SNAPPY always consumes the whole buffer or throws an exception src.position(src.limit()); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java index 07afbab7246b7..7b3099819497f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java @@ -26,7 +26,7 @@ import org.apache.hadoop.io.compress.zlib.ZlibCompressor.CompressionStrategy; import org.apache.hadoop.util.NativeCodeLoader; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java index 7445502c989d8..8b1b6db086021 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java @@ -18,7 +18,7 @@ package org.apache.hadoop.io.compress.zstd; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -159,7 +159,7 @@ public boolean needsInput() { } // have we consumed all input - if (keepUncompressedBuf && uncompressedDirectBufLen > 0) { + if (keepUncompressedBuf && uncompressedDirectBufLen - uncompressedDirectBufOff > 0) { return false; } @@ -223,7 +223,7 @@ public int compress(byte[] b, int off, int len) throws IOException { compressedDirectBuf.limit(n); // Check if we have consumed all input buffer - if (uncompressedDirectBufLen <= 0) { + if (uncompressedDirectBufLen - uncompressedDirectBufOff <= 0) { // consumed all input buffer keepUncompressedBuf = false; uncompressedDirectBuf.clear(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecRegistry.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecRegistry.java index daf91e22ecc91..359e07e27fd2a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecRegistry.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecRegistry.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.io.erasurecode; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java index 5ba6e9c0dd1ed..c871ce3067e00 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.io.erasurecode; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.erasurecode.codec.ErasureCodec; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlockGroup.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlockGroup.java index c52d9471bd576..9dc3a3e777a16 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlockGroup.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlockGroup.java @@ -54,34 +54,6 @@ public ECBlock[] getParityBlocks() { return parityBlocks; } - /** - * Any erased data block? - * @return true if any erased data block, false otherwise - */ - public boolean anyErasedDataBlock() { - for (int i = 0; i < dataBlocks.length; ++i) { - if (dataBlocks[i].isErased()) { - return true; - } - } - - return false; - } - - /** - * Any erased parity block? - * @return true if any erased parity block, false otherwise - */ - public boolean anyErasedParityBlock() { - for (int i = 0; i < parityBlocks.length; ++i) { - if (parityBlocks[i].isErased()) { - return true; - } - } - - return false; - } - /** * Get erased blocks count * @return erased count of blocks diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java index f5122033465d3..27e1b78919ceb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java @@ -156,7 +156,7 @@ public String getCodecName() { } /** - * Get extra options specific to a erasure code. + * Get extra options specific to an erasure code. * @return extra options */ public Map getExtraOptions() { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java index faf44d944eeaf..004fd38df1141 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java @@ -172,27 +172,4 @@ protected int[] getErasedIndexes(ECBlock[] inputBlocks) { return erasedIndexes; } - - /** - * Get erased input blocks from inputBlocks - * @param inputBlocks - * @return an array of erased blocks from inputBlocks - */ - protected ECBlock[] getErasedBlocks(ECBlock[] inputBlocks) { - int numErased = getNumErasedBlocks(inputBlocks); - if (numErased == 0) { - return new ECBlock[0]; - } - - ECBlock[] erasedBlocks = new ECBlock[numErased]; - int i = 0, j = 0; - for (; i < inputBlocks.length && j < erasedBlocks.length; i++) { - if (inputBlocks[i].isErased()) { - erasedBlocks[j++] = inputBlocks[i]; - } - } - - return erasedBlocks; - } - } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java index c4347e0c1afab..6eee025a2339d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java @@ -24,7 +24,7 @@ import java.io.OutputStream; import java.util.ArrayList; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.compress.CodecPool; import org.apache.hadoop.io.compress.CompressionCodec; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java index 160b8e029e56b..06eb7a68aba72 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java @@ -47,7 +47,7 @@ import org.slf4j.LoggerFactory; import sun.misc.Unsafe; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * JNI wrappers for various native IO-related calls not available in Java. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/AsyncCallHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/AsyncCallHandler.java index 69e1233e16b07..d4921f7f57993 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/AsyncCallHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/AsyncCallHandler.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.io.retry; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.ipc.Client; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/CallReturn.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/CallReturn.java index 022b78507fcee..7ccd6deb7f913 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/CallReturn.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/CallReturn.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.io.retry; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** The call return from a method invocation. */ class CallReturn { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java index 6db00d724aa35..63e45ed034f6f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.io.retry; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.io.retry.FailoverProxyProvider.ProxyInfo; import org.apache.hadoop.io.retry.RetryPolicy.RetryAction; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java index fcbcc868cf6dd..e6ccd7671b631 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java @@ -43,7 +43,7 @@ import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.ietf.jgss.GSSException; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java index 53ac34b61272f..7a17ced812efc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java @@ -33,7 +33,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcStatusProto; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java index dd14ba133e124..378b83d13b0c7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java @@ -21,10 +21,17 @@ import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY; /** * A class defining the caller context for auditing coarse granularity @@ -54,8 +61,8 @@ public final class CallerContext { private final byte[] signature; private CallerContext(Builder builder) { - this.context = builder.context; - this.signature = builder.signature; + this.context = builder.getContext(); + this.signature = builder.getSignature(); } public String getContext() { @@ -109,11 +116,61 @@ public String toString() { /** The caller context builder. */ public static final class Builder { - private final String context; + private static final String KEY_VALUE_SEPARATOR = ":"; + /** + * The illegal separators include '\t', '\n', '='. + * User should not set illegal separator. + */ + private static final Set ILLEGAL_SEPARATORS = + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("\t", "\n", "="))); + private final String fieldSeparator; + private final StringBuilder sb = new StringBuilder(); private byte[] signature; public Builder(String context) { - this.context = context; + this(context, HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT); + } + + public Builder(String context, Configuration conf) { + if (isValid(context)) { + sb.append(context); + } + fieldSeparator = conf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, + HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT); + checkFieldSeparator(fieldSeparator); + } + + public Builder(String context, String separator) { + if (isValid(context)) { + sb.append(context); + } + fieldSeparator = separator; + checkFieldSeparator(fieldSeparator); + } + + /** + * Check whether the separator is legal. + * The illegal separators include '\t', '\n', '='. + * Throw IllegalArgumentException if the separator is Illegal. + * @param separator the separator of fields. + */ + private void checkFieldSeparator(String separator) { + if (ILLEGAL_SEPARATORS.contains(separator)) { + throw new IllegalArgumentException("Illegal field separator: " + + separator); + } + } + + /** + * Whether the field is valid. + * The field should not contain '\t', '\n', '='. + * Because the context could be written to audit log. + * @param field one of the fields in context. + * @return true if the field is not null or empty. + */ + private boolean isValid(String field) { + return field != null && field.length() > 0; } public Builder setSignature(byte[] signature) { @@ -123,6 +180,54 @@ public Builder setSignature(byte[] signature) { return this; } + /** + * Get the context. + * For example, the context is "key1:value1,key2:value2". + * @return the valid context or null. + */ + public String getContext() { + return sb.length() > 0 ? sb.toString() : null; + } + + /** + * Get the signature. + * @return the signature. + */ + public byte[] getSignature() { + return signature; + } + + /** + * Append new field to the context. + * @param field one of fields to append. + * @return the builder. + */ + public Builder append(String field) { + if (isValid(field)) { + if (sb.length() > 0) { + sb.append(fieldSeparator); + } + sb.append(field); + } + return this; + } + + /** + * Append new field which contains key and value to the context. + * @param key the key of field. + * @param value the value of field. + * @return the builder. + */ + public Builder append(String key, String value) { + if (isValid(key) && isValid(value)) { + if (sb.length() > 0) { + sb.append(fieldSeparator); + } + sb.append(key).append(KEY_VALUE_SEPARATOR).append(value); + } + return this; + } + public CallerContext build() { return new CallerContext(this); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index 6240f859cf786..ae3999b775bcd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -18,9 +18,9 @@ package org.apache.hadoop.ipc; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java index a0720d421820a..94a0f04060747 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java @@ -29,6 +29,8 @@ import org.apache.hadoop.io.ObjectWritable; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; + /* Cache a client using its socket factory as the hash key */ @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) @InterfaceStability.Evolving @@ -114,4 +116,10 @@ public void stopClient(Client client) { client.stop(); } } + + @VisibleForTesting + public void clearCache() { + clients.values().forEach(c -> c.stop()); + clients.clear(); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientId.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientId.java index c95dbb9facaac..cc6581618b5f1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientId.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientId.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * A class defining a set of static helper methods to provide conversion between diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java index 45cbd4e99dff8..97baa65d8787a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java @@ -20,11 +20,14 @@ import java.lang.ref.WeakReference; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; @@ -37,8 +40,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.AtomicDoubleArray; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.AtomicDoubleArray; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -53,7 +56,7 @@ import org.apache.hadoop.metrics2.util.Metrics2Util.NameValuePair; import org.apache.hadoop.metrics2.util.Metrics2Util.TopN; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,6 +111,13 @@ public class DecayRpcScheduler implements RpcScheduler, public static final String IPC_FCQ_DECAYSCHEDULER_THRESHOLDS_KEY = "faircallqueue.decay-scheduler.thresholds"; + /** + * Service users will always be scheduled into the highest-priority queue. + * They are specified as a comma-separated list. + */ + public static final String IPC_DECAYSCHEDULER_SERVICE_USERS_KEY = + "decay-scheduler.service-users"; + // Specifies the identity to use when the IdentityProvider cannot handle // a schedulable. public static final String DECAYSCHEDULER_UNKNOWN_IDENTITY = @@ -142,10 +152,15 @@ public class DecayRpcScheduler implements RpcScheduler, private final ConcurrentHashMap> callCosts = new ConcurrentHashMap>(); - // Should be the sum of all AtomicLongs in decayed callCosts + // Should be the sum of all AtomicLongs in decayed callCosts except + // service-user. private final AtomicLong totalDecayedCallCost = new AtomicLong(); - // The sum of all AtomicLongs in raw callCosts + // The sum of all AtomicLongs in raw callCosts except service-user. private final AtomicLong totalRawCallCost = new AtomicLong(); + // Should be the sum of all AtomicLongs in decayed callCosts of service-user. + private final AtomicLong totalServiceUserDecayedCallCost = new AtomicLong(); + // The sum of all AtomicLongs in raw callCosts of service-user. + private final AtomicLong totalServiceUserRawCallCost = new AtomicLong(); // Track total call count and response time in current decay window @@ -178,6 +193,7 @@ public class DecayRpcScheduler implements RpcScheduler, private static final double PRECISION = 0.0001; private MetricsProxy metricsProxy; private final CostProvider costProvider; + private Set serviceUserNames; /** * This TimerTask will call decayCurrentCosts until @@ -229,6 +245,7 @@ public DecayRpcScheduler(int numLevels, String ns, Configuration conf) { conf); this.backOffResponseTimeThresholds = parseBackOffResponseTimeThreshold(ns, conf, numLevels); + this.serviceUserNames = this.parseServiceUserNames(ns, conf); // Setup response time metrics responseTimeTotalInCurrWindow = new AtomicLongArray(numLevels); @@ -359,6 +376,12 @@ private static double[] parseThresholds(String ns, Configuration conf, return decimals; } + private Set parseServiceUserNames(String ns, Configuration conf) { + Collection collection = conf.getStringCollection( + ns + "." + IPC_DECAYSCHEDULER_SERVICE_USERS_KEY); + return new HashSet<>(collection); + } + /** * Generate default thresholds if user did not specify. Strategy is * to halve each time, since queue usage tends to be exponential. @@ -428,6 +451,8 @@ private void decayCurrentCosts() { try { long totalDecayedCost = 0; long totalRawCost = 0; + long totalServiceUserDecayedCost = 0; + long totalServiceUserRawCost = 0; Iterator>> it = callCosts.entrySet().iterator(); @@ -438,10 +463,15 @@ private void decayCurrentCosts() { // Compute the next value by reducing it by the decayFactor - totalRawCost += rawCost.get(); long currentValue = decayedCost.get(); long nextValue = (long) (currentValue * decayFactor); - totalDecayedCost += nextValue; + if (isServiceUser((String) entry.getKey())) { + totalServiceUserRawCost += rawCost.get(); + totalServiceUserDecayedCost += nextValue; + } else { + totalRawCost += rawCost.get(); + totalDecayedCost += nextValue; + } decayedCost.set(nextValue); LOG.debug( @@ -460,9 +490,13 @@ private void decayCurrentCosts() { // Update the total so that we remain in sync totalDecayedCallCost.set(totalDecayedCost); totalRawCallCost.set(totalRawCost); + totalServiceUserDecayedCallCost.set(totalServiceUserDecayedCost); + totalServiceUserRawCallCost.set(totalServiceUserRawCost); - LOG.debug("After decaying the stored costs, totalDecayedCost: {}, " + - "totalRawCallCost: {}.", totalDecayedCost, totalRawCost); + LOG.debug("After decaying the stored costs, totalDecayedCost: {}, " + + "totalRawCallCost: {}, totalServiceUserDecayedCost: {}," + + " totalServiceUserRawCost: {}.", totalDecayedCost, totalRawCost, + totalServiceUserDecayedCost, totalServiceUserRawCost); // Now refresh the cache of scheduling decisions recomputeScheduleCache(); @@ -486,7 +520,7 @@ private void recomputeScheduleCache() { AtomicLong value = entry.getValue().get(0); long snapshot = value.get(); - int computedLevel = computePriorityLevel(snapshot); + int computedLevel = computePriorityLevel(snapshot, id); nextCache.put(id, computedLevel); } @@ -520,8 +554,13 @@ private void addCost(Object identity, long costDelta) { } // Update the total - totalDecayedCallCost.getAndAdd(costDelta); - totalRawCallCost.getAndAdd(costDelta); + if (!isServiceUser((String) identity)) { + totalDecayedCallCost.getAndAdd(costDelta); + totalRawCallCost.getAndAdd(costDelta); + } else { + totalServiceUserDecayedCallCost.getAndAdd(costDelta); + totalServiceUserRawCallCost.getAndAdd(costDelta); + } // At this point value is guaranteed to be not null. It may however have // been clobbered from callCosts. Nonetheless, we return what @@ -534,9 +573,15 @@ private void addCost(Object identity, long costDelta) { * Given the cost for an identity, compute a scheduling decision. * * @param cost the cost for an identity + * @param identity the identity of the user * @return scheduling decision from 0 to numLevels - 1 */ - private int computePriorityLevel(long cost) { + private int computePriorityLevel(long cost, Object identity) { + // The priority for service users is always 0 + if (isServiceUser((String)identity)) { + return 0; + } + long totalCallSnapshot = totalDecayedCallCost.get(); double proportion = 0; @@ -576,7 +621,7 @@ private int cachedOrComputedPriorityLevel(Object identity) { // Cache was no good, compute it List costList = callCosts.get(identity); long currentCost = costList == null ? 0 : costList.get(0).get(); - int priority = computePriorityLevel(currentCost); + int priority = computePriorityLevel(currentCost, identity); LOG.debug("compute priority for {} priority {}", identity, priority); return priority; } @@ -598,6 +643,10 @@ public int getPriorityLevel(Schedulable obj) { return cachedOrComputedPriorityLevel(identity); } + private boolean isServiceUser(String userName) { + return this.serviceUserNames.contains(userName); + } + @Override public boolean shouldBackOff(Schedulable obj) { Boolean backOff = false; @@ -698,6 +747,11 @@ long getDecayPeriodMillis() { return thresholds; } + @VisibleForTesting + Set getServiceUserNames() { + return serviceUserNames; + } + @VisibleForTesting void forceDecay() { decayCurrentCosts(); @@ -860,6 +914,14 @@ public long getTotalRawCallVolume() { return totalRawCallCost.get(); } + public long getTotalServiceUserCallVolume() { + return totalServiceUserDecayedCallCost.get(); + } + + public long getTotalServiceUserRawCallVolume() { + return totalServiceUserRawCallCost.get(); + } + public long[] getResponseTimeCountInLastWindow() { long[] ret = new long[responseTimeCountInLastWindow.length()]; for (int i = 0; i < responseTimeCountInLastWindow.length(); i++) { @@ -889,6 +951,8 @@ public void getMetrics(MetricsCollector collector, boolean all) { addAvgResponseTimePerPriority(rb); addCallVolumePerPriority(rb); addRawCallVolume(rb); + addServiceUserDecayedCallVolume(rb); + addServiceUserRawCallVolume(rb); } catch (Exception e) { LOG.warn("Exception thrown while metric collection. Exception : " + e.getMessage()); @@ -912,6 +976,20 @@ private void addRawCallVolume(MetricsRecordBuilder rb) { "incoming Call Volume"), getTotalRawCallVolume()); } + // Key: ServiceUserDecayedCallVolume. + private void addServiceUserDecayedCallVolume(MetricsRecordBuilder rb) { + rb.addCounter(Interns.info("ServiceUserDecayedCallVolume", + "Service-user Decayed Total incoming Call Volume"), + getTotalServiceUserCallVolume()); + } + + // Key: ServiceUserCallVolume. + private void addServiceUserRawCallVolume(MetricsRecordBuilder rb) { + rb.addCounter(Interns.info("ServiceUserCallVolume", + "Service-user Raw Total incoming Call Volume"), + getTotalServiceUserRawCallVolume()); + } + // Key: Priority.0.CompletedCallVolume private void addCallVolumePerPriority(MetricsRecordBuilder rb) { for (int i = 0; i < responseTimeCountInLastWindow.length(); i++) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java index 939149fcc5e57..6db9348980159 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java @@ -31,7 +31,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.NotImplementedException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java index 220ad1ded9fec..b7b7ad4db65cd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ipc; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import com.google.protobuf.BlockingService; import com.google.protobuf.Descriptors.MethodDescriptor; import com.google.protobuf.Message; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java index 30315343962c8..5043051ce0a2c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ipc; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.protobuf.*; import org.apache.hadoop.thirdparty.protobuf.Descriptors.MethodDescriptor; import org.apache.hadoop.classification.InterfaceAudience; @@ -352,6 +352,11 @@ public RPC.Server getServer(Class protocol, Object protocolImpl, portRangeConfig, alignmentContext); } + @VisibleForTesting + public static void clearClientCache() { + CLIENTS.clearCache(); + } + public static class Server extends RPC.Server { static final ThreadLocal CURRENT_CALLBACK = diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolSignature.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolSignature.java index 69d18eacd34d6..e1a1f0eb3e7a1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolSignature.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolSignature.java @@ -29,7 +29,7 @@ import org.apache.hadoop.io.WritableFactories; import org.apache.hadoop.io.WritableFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; public class ProtocolSignature implements Writable { static { // register a ctor diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProxyCombiner.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProxyCombiner.java index 99eb487be495c..b7188b7de59e2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProxyCombiner.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProxyCombiner.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.ipc; -import com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import java.io.Closeable; import java.io.IOException; import java.lang.reflect.InvocationHandler; @@ -50,8 +50,8 @@ private ProxyCombiner() { } * all of the methods of the combined proxy interface, delegating calls * to which proxy implements that method. If multiple proxies implement the * same method, the first in the list will be used for delegation. - * - *

This will check that every method on the combined interface is + *

+ * This will check that every method on the combined interface is * implemented by at least one of the supplied proxy objects. * * @param combinedProxyInterface The interface of the combined proxy. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java index e794cb913c232..ad3628d01855d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java @@ -195,14 +195,18 @@ private RPC() {} // no public ctor private static final String ENGINE_PROP = "rpc.engine"; /** - * Set a protocol to use a non-default RpcEngine. + * Set a protocol to use a non-default RpcEngine if one + * is not specified in the configuration. * @param conf configuration to use * @param protocol the protocol interface * @param engine the RpcEngine impl */ public static void setProtocolEngine(Configuration conf, Class protocol, Class engine) { - conf.setClass(ENGINE_PROP+"."+protocol.getName(), engine, RpcEngine.class); + if (conf.get(ENGINE_PROP+"."+protocol.getName()) == null) { + conf.setClass(ENGINE_PROP+"."+protocol.getName(), engine, + RpcEngine.class); + } } // return the RpcEngine configured to handle a protocol diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshRegistry.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshRegistry.java index e67e8d9cbeb92..95f1323410fe0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshRegistry.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshRegistry.java @@ -20,9 +20,9 @@ import java.util.ArrayList; import java.util.Collection; -import com.google.common.base.Joiner; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultimap; +import org.apache.hadoop.thirdparty.com.google.common.collect.Multimap; import org.apache.hadoop.classification.InterfaceStability; import org.slf4j.Logger; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java index 4bde261eab1b0..d71edded9caec 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java @@ -28,8 +28,8 @@ import org.apache.hadoop.util.LightWeightGSet; import org.apache.hadoop.util.LightWeightGSet.LinkedElement; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index 907d55f9be347..3afad2135f091 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -123,7 +123,7 @@ import org.apache.htrace.core.Tracer; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.hadoop.thirdparty.protobuf.Message; @@ -2043,7 +2043,7 @@ private void saslProcess(RpcSaslProto saslMessage) LOG.debug("SASL server successfully authenticated client: " + user); } rpcMetrics.incrAuthenticationSuccesses(); - AUDITLOG.info(AUTH_SUCCESSFUL_FOR + user); + AUDITLOG.info(AUTH_SUCCESSFUL_FOR + user + " from " + toString()); saslContextEstablished = true; } } catch (RpcServerException rse) { // don't re-wrap @@ -3714,7 +3714,7 @@ void incrUserConnections(String user) { if (count == null) { count = 1; } else { - count++; + count = count + 1; } userToConnectionsMap.put(user, count); } @@ -3726,7 +3726,7 @@ void decrUserConnections(String user) { if (count == null) { return; } else { - count--; + count = count - 1; } if (count == 0) { userToConnectionsMap.remove(user); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WeightedTimeCostProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WeightedTimeCostProvider.java index 4304b24299f29..1ecd19b74c45a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WeightedTimeCostProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WeightedTimeCostProvider.java @@ -29,8 +29,8 @@ * {@link ProcessingDetails}). This can be used by specifying the * {@link org.apache.hadoop.fs.CommonConfigurationKeys#IPC_COST_PROVIDER_KEY} * configuration key. - * - *

This allows for configuration of how heavily each of the operations + *

+ * This allows for configuration of how heavily each of the operations * within {@link ProcessingDetails} is weighted. By default, * {@link ProcessingDetails.Timing#LOCKFREE}, * {@link ProcessingDetails.Timing#RESPONSE}, and diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/DecayRpcSchedulerDetailedMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/DecayRpcSchedulerDetailedMetrics.java index 04a6c0eab1c42..ce65e9c64622d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/DecayRpcSchedulerDetailedMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/DecayRpcSchedulerDetailedMetrics.java @@ -27,7 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * This class is for maintaining queue (priority) level related diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java index bb4bfcfd08be5..439b87326c2ee 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java @@ -19,7 +19,7 @@ import java.util.concurrent.TimeUnit; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java index 79eae12314493..829d276ea7281 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java @@ -32,8 +32,8 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Jdk14Logger; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java index eb147ca47ee19..6bf75f9d00a7c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.log; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.util.HashMap; import java.util.Map; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/AbstractMetric.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/AbstractMetric.java index e2574f647e3a2..a9e777bcba952 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/AbstractMetric.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/AbstractMetric.java @@ -18,13 +18,13 @@ package org.apache.hadoop.metrics2; -import com.google.common.base.Objects; +import org.apache.hadoop.thirdparty.com.google.common.base.Objects; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import java.util.StringJoiner; -import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull; /** * The immutable metric diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsTag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsTag.java index db8a5d9a8578e..26973f8fb9870 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsTag.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsTag.java @@ -18,13 +18,13 @@ package org.apache.hadoop.metrics2; -import com.google.common.base.Objects; +import org.apache.hadoop.thirdparty.com.google.common.base.Objects; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import java.util.StringJoiner; -import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull; /** * Immutable tag for metrics (for grouping on host/queue/username etc.) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/filter/AbstractPatternFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/filter/AbstractPatternFilter.java index ca1d7f97f340c..7e3257d409ebc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/filter/AbstractPatternFilter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/filter/AbstractPatternFilter.java @@ -20,7 +20,7 @@ import java.util.Map; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/AbstractMetricsRecord.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/AbstractMetricsRecord.java index a4632c60c108e..28348c7ae36b8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/AbstractMetricsRecord.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/AbstractMetricsRecord.java @@ -18,8 +18,8 @@ package org.apache.hadoop.metrics2.impl; -import com.google.common.base.Objects; -import com.google.common.collect.Iterables; +import org.apache.hadoop.thirdparty.com.google.common.base.Objects; +import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables; import org.apache.hadoop.metrics2.MetricsRecord; import java.util.StringJoiner; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MBeanInfoBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MBeanInfoBuilder.java index cdd0ba4275ce2..a297072d236d4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MBeanInfoBuilder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MBeanInfoBuilder.java @@ -22,7 +22,7 @@ import javax.management.MBeanAttributeInfo; import javax.management.MBeanInfo; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsInfo; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsCollectorImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsCollectorImpl.java index 5345c1baf88fd..4b4b70bd8e607 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsCollectorImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsCollectorImpl.java @@ -21,8 +21,8 @@ import java.util.Iterator; import java.util.List; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsInfo; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java index 976f16bedd81b..2d22b75841b33 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java @@ -29,14 +29,15 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Splitter; +import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.commons.configuration2.Configuration; import org.apache.commons.configuration2.PropertiesConfiguration; import org.apache.commons.configuration2.SubsetConfiguration; +import org.apache.commons.configuration2.convert.DefaultListDelimiterHandler; import org.apache.commons.configuration2.ex.ConfigurationException; import org.apache.commons.configuration2.io.FileHandler; import org.apache.hadoop.metrics2.MetricsFilter; @@ -111,6 +112,7 @@ static MetricsConfig loadFirst(String prefix, String... fileNames) { for (String fname : fileNames) { try { PropertiesConfiguration pcf = new PropertiesConfiguration(); + pcf.setListDelimiterHandler(new DefaultListDelimiterHandler(',')); FileHandler fh = new FileHandler(pcf); fh.setFileName(fname); fh.load(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordBuilderImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordBuilderImpl.java index f66ec5cb99ffb..19e4c3b6d4187 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordBuilderImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordBuilderImpl.java @@ -21,7 +21,7 @@ import java.util.Collections; import java.util.List; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsInfo; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordFiltered.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordFiltered.java index 5674dfbf68d26..58ebbcffa8e58 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordFiltered.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordFiltered.java @@ -21,7 +21,7 @@ import java.util.Iterator; import java.util.Collection; -import com.google.common.collect.AbstractIterator; +import org.apache.hadoop.thirdparty.com.google.common.collect.AbstractIterator; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsFilter; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java index 8eb6cb8f8678a..14b930e830d77 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java @@ -20,7 +20,7 @@ import java.util.List; -import static com.google.common.base.Preconditions.*; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.AbstractMetric; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSinkAdapter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSinkAdapter.java index f2e607b577619..836d9d5cf816f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSinkAdapter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSinkAdapter.java @@ -22,7 +22,7 @@ import java.util.Random; import java.util.concurrent.*; -import static com.google.common.base.Preconditions.*; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.metrics2.lib.MutableGaugeInt; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java index f12ec67dd6125..852f31995a27b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java @@ -29,10 +29,10 @@ import javax.management.ObjectName; import javax.management.ReflectionException; -import static com.google.common.base.Preconditions.*; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsFilter; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java index cf4b4a9810c4f..a6edf08e5a717 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java @@ -30,10 +30,10 @@ import java.util.TimerTask; import javax.management.ObjectName; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.annotations.VisibleForTesting; -import static com.google.common.base.Preconditions.*; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*; import org.apache.commons.configuration2.PropertiesConfiguration; import org.apache.commons.math3.util.ArithmeticUtils; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java index 935f47f3a09ee..83e458f06c68c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java @@ -27,7 +27,7 @@ import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.impl.MetricsSystemImpl; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * The default metrics system singleton. This class is used by all the daemon diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MethodMetric.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MethodMetric.java index 9b54adcb4314f..96eb5026be179 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MethodMetric.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MethodMetric.java @@ -20,7 +20,7 @@ import java.lang.reflect.Method; -import static com.google.common.base.Preconditions.*; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.metrics2.MetricsException; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java index e3adc821de5c4..e86398f544edf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java @@ -18,12 +18,12 @@ package org.apache.hadoop.metrics2.lib; -import com.google.common.base.Objects; +import org.apache.hadoop.thirdparty.com.google.common.base.Objects; import org.apache.hadoop.metrics2.MetricsInfo; import java.util.StringJoiner; -import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull; /** * Making implementing metric info a little easier diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java index 6227d0954fb6d..b71f7f8cc5ee0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java @@ -18,7 +18,7 @@ package org.apache.hadoop.metrics2.lib; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.metrics2.MetricsException; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java index 1fcede464dacd..f400f02d256f2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java @@ -22,7 +22,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; -import static com.google.common.base.Preconditions.*; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsCollector; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounter.java index 8ba72343f2249..e616bb6d934dd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounter.java @@ -18,7 +18,7 @@ package org.apache.hadoop.metrics2.lib; -import static com.google.common.base.Preconditions.*; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGauge.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGauge.java index 03384f69147b7..6c77e97353869 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGauge.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGauge.java @@ -18,7 +18,7 @@ package org.apache.hadoop.metrics2.lib; -import static com.google.common.base.Preconditions.*; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java index 910805e4246d0..0e69c268c94cd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java @@ -35,8 +35,8 @@ import org.apache.hadoop.metrics2.util.QuantileEstimator; import org.apache.hadoop.metrics2.util.SampleQuantiles; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; /** * Watches a stream of long values, maintaining online estimates of specific diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRates.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRates.java index 994eb13e08dae..c31c2e67f8f31 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRates.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRates.java @@ -21,8 +21,8 @@ import java.lang.reflect.Method; import java.util.Set; -import static com.google.common.base.Preconditions.*; -import com.google.common.collect.Sets; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java index 5fe0083aa5dce..207916589f694 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java @@ -18,7 +18,7 @@ package org.apache.hadoop.metrics2.lib; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import java.lang.ref.WeakReference; import java.lang.reflect.Method; import java.util.Iterator; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java index 6803d11d1ca38..e6111e36bb76d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java @@ -31,7 +31,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -39,8 +39,8 @@ import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import javax.annotation.Nullable; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/UniqueNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/UniqueNames.java index 0df852d46b371..2508ee27bbee8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/UniqueNames.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/UniqueNames.java @@ -20,8 +20,8 @@ import java.util.Map; -import com.google.common.base.Joiner; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java index 1d330c74ab46a..4dfe9c6854049 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java @@ -18,7 +18,7 @@ package org.apache.hadoop.metrics2.sink; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.io.Closeable; import java.io.IOException; import java.io.PrintStream; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java index f19a2be0b4195..816940b109879 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java @@ -27,8 +27,8 @@ import java.util.List; import java.util.concurrent.ConcurrentHashMap; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java index 4aef03a5e645f..afde91113cc5d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java @@ -28,12 +28,12 @@ import javax.management.MBeanServer; import javax.management.ObjectName; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MetricsCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MetricsCache.java index 6cfbc39f896f4..bf6e910c3e232 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MetricsCache.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MetricsCache.java @@ -18,7 +18,7 @@ package org.apache.hadoop.metrics2.util; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.metrics2.AbstractMetric; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java index 40e31bd841ba8..7a100edc228fb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java @@ -20,7 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience; -import com.google.common.collect.ComparisonChain; +import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; /** * Specifies a quantile (with error bounds) to be watched by a diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java index 0c5d98f2374ed..7383b2d90d70b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java @@ -26,9 +26,9 @@ import org.apache.hadoop.classification.InterfaceAudience; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Implementation of the Cormode, Korn, Muthukrishnan, and Srivastava algorithm diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Servers.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Servers.java index 19a64a9f9d171..2bd49e9f211ba 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Servers.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Servers.java @@ -22,7 +22,7 @@ import java.net.InetSocketAddress; import java.util.List; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java index 531ad80f41722..ffe7ee5676aee 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java @@ -18,7 +18,7 @@ package org.apache.hadoop.net; -import com.google.common.net.InetAddresses; +import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.slf4j.Logger; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DomainNameResolverFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DomainNameResolverFactory.java index a0b0380c18963..fdb45dd85d962 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DomainNameResolverFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DomainNameResolverFactory.java @@ -22,7 +22,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ReflectionUtils; -import java.io.IOException; import java.net.URI; /** @@ -49,7 +48,7 @@ private DomainNameResolverFactory() { * @return Domain name resolver. */ public static DomainNameResolver newInstance( - Configuration conf, URI uri, String configKey) throws IOException { + Configuration conf, URI uri, String configKey) { String host = uri.getHost(); String confKeyWithHost = configKey + "." + host; return newInstance(conf, confKeyWithHost); @@ -61,7 +60,6 @@ public static DomainNameResolver newInstance( * @param conf Configuration * @param configKey config key name. * @return Domain name resolver. - * @throws IOException when the class cannot be found or initiated. */ public static DomainNameResolver newInstance( Configuration conf, String configKey) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java index c5a5b111b2b3d..2b2c028552483 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java @@ -39,12 +39,16 @@ import java.nio.channels.SocketChannel; import java.nio.channels.UnresolvedAddressException; import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import javax.net.SocketFactory; +import org.apache.hadoop.thirdparty.com.google.common.cache.Cache; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; + import org.apache.commons.net.util.SubnetUtils; import org.apache.commons.net.util.SubnetUtils.SubnetInfo; import org.apache.hadoop.classification.InterfaceAudience; @@ -56,7 +60,7 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.util.ReflectionUtils; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -177,11 +181,33 @@ public static InetSocketAddress createSocketAddr(String target, * include a port number * @param configName the name of the configuration from which * target was loaded. This is used in the - * exception message in the case that parsing fails. + * exception message in the case that parsing fails. */ public static InetSocketAddress createSocketAddr(String target, int defaultPort, String configName) { + return createSocketAddr(target, defaultPort, configName, false); + } + + /** + * Create an InetSocketAddress from the given target string and + * default port. If the string cannot be parsed correctly, the + * configName parameter is used as part of the + * exception message, allowing the user to better diagnose + * the misconfiguration. + * + * @param target a string of either "host" or "host:port" + * @param defaultPort the default port if target does not + * include a port number + * @param configName the name of the configuration from which + * target was loaded. This is used in the + * exception message in the case that parsing fails. + * @param useCacheIfPresent Whether use cache when create URI + */ + public static InetSocketAddress createSocketAddr(String target, + int defaultPort, + String configName, + boolean useCacheIfPresent) { String helpText = ""; if (configName != null) { helpText = " (configuration property '" + configName + "')"; @@ -191,15 +217,8 @@ public static InetSocketAddress createSocketAddr(String target, helpText); } target = target.trim(); - boolean hasScheme = target.contains("://"); - URI uri = null; - try { - uri = hasScheme ? URI.create(target) : URI.create("dummyscheme://"+target); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException( - "Does not contain a valid host:port authority: " + target + helpText - ); - } + boolean hasScheme = target.contains("://"); + URI uri = createURI(target, hasScheme, helpText, useCacheIfPresent); String host = uri.getHost(); int port = uri.getPort(); @@ -207,10 +226,9 @@ public static InetSocketAddress createSocketAddr(String target, port = defaultPort; } String path = uri.getPath(); - + if ((host == null) || (port < 0) || - (!hasScheme && path != null && !path.isEmpty())) - { + (!hasScheme && path != null && !path.isEmpty())) { throw new IllegalArgumentException( "Does not contain a valid host:port authority: " + target + helpText ); @@ -218,6 +236,40 @@ public static InetSocketAddress createSocketAddr(String target, return createSocketAddrForHost(host, port); } + private static final long URI_CACHE_SIZE_DEFAULT = 1000; + private static final long URI_CACHE_EXPIRE_TIME_DEFAULT = 12; + private static final Cache URI_CACHE = CacheBuilder.newBuilder() + .maximumSize(URI_CACHE_SIZE_DEFAULT) + .expireAfterWrite(URI_CACHE_EXPIRE_TIME_DEFAULT, TimeUnit.HOURS) + .build(); + + private static URI createURI(String target, + boolean hasScheme, + String helpText, + boolean useCacheIfPresent) { + URI uri; + if (useCacheIfPresent) { + uri = URI_CACHE.getIfPresent(target); + if (uri != null) { + return uri; + } + } + + try { + uri = hasScheme ? URI.create(target) : + URI.create("dummyscheme://" + target); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Does not contain a valid host:port authority: " + target + helpText + ); + } + + if (useCacheIfPresent) { + URI_CACHE.put(target, uri); + } + return uri; + } + /** * Create a socket address with the given host and port. The hostname * might be replaced with another host that was set via diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java index 893012befcf44..58b2f0bfda318 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.net; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputWrapper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputWrapper.java index f5cbe17519d60..45f776e692ac4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputWrapper.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputWrapper.java @@ -27,7 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * A wrapper stream around a socket which allows setting of its timeout. If the diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java index 9693220438dd6..99fde5cb50407 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java @@ -32,7 +32,7 @@ import org.apache.hadoop.util.NativeCodeLoader; import org.apache.hadoop.util.CloseableReferenceCount; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java index e36399ff96c01..17c7d4b65c401 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java @@ -35,9 +35,9 @@ import org.apache.commons.lang3.SystemUtils; import org.apache.hadoop.util.NativeCodeLoader; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java index 961ec7d591924..47dca6cfe94fd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java @@ -36,18 +36,18 @@ import org.apache.htrace.core.TraceScope; import org.apache.htrace.core.Tracer; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Ticker; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Ticker; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.Cache; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; +import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; @@ -201,10 +201,10 @@ private IOException noGroupsForUser(String user) { /** * Get the group memberships of a given user. * If the user's group is not cached, this method may block. - * Note this method can be expensive as it involves Set->List conversion. - * For user with large group membership (i.e., > 1000 groups), we recommend - * using getGroupSet to avoid the conversion and fast membership look up via - * contains(). + * Note this method can be expensive as it involves Set {@literal ->} List + * conversion. For user with large group membership + * (i.e., {@literal >} 1000 groups), we recommend using getGroupSet + * to avoid the conversion and fast membership look up via contains(). * @param user User's name * @return the group memberships of the user as list * @throws IOException if user does not exist @@ -220,7 +220,9 @@ public List getGroups(final String user) throws IOException { * Get the group memberships of a given user. * If the user's group is not cached, this method may block. * This provide better performance when user has large group membership via - * 1) avoid set->list->set conversion for the caller UGI/PermissionCheck + *
+ * 1) avoid {@literal set->list->set} conversion for the caller + * UGI/PermissionCheck
* 2) fast lookup using contains() via Set instead of List * @param user User's name * @return the group memberships of the user as set diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IngressPortBasedResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IngressPortBasedResolver.java index a30e4a84dd86b..1431ed5d0e907 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IngressPortBasedResolver.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IngressPortBasedResolver.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.security; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.net.InetAddress; import java.util.Collection; import java.util.HashMap; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java index 3f656990517af..10483a2c56571 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java @@ -59,7 +59,7 @@ import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; -import com.google.common.collect.Iterators; +import org.apache.hadoop.thirdparty.com.google.common.collect.Iterators; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java index d7a68210c7dc2..23efffcd98b0a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java @@ -24,7 +24,7 @@ import java.net.URISyntaxException; import java.net.URL; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java index 215f473b9fcc9..7dc28fb277aa8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java @@ -71,7 +71,7 @@ import org.apache.hadoop.security.token.TokenSelector; import org.apache.hadoop.util.ProtoUtil; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.protobuf.ByteString; import com.google.re2j.Pattern; import org.slf4j.Logger; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java index aa12b93be95b4..81f8143f11849 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java @@ -57,8 +57,8 @@ import org.xbill.DNS.ResolverConfig; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.net.InetAddresses; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses; /** * Security Utils. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java index 93231075282da..0ce529ea84656 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java @@ -33,9 +33,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.Time; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.BiMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.HashBiMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java index f3432a6f91762..ca9f474cb2670 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java @@ -26,8 +26,8 @@ import java.util.StringTokenizer; import java.util.concurrent.TimeUnit; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index 5269e5a33061a..f5007588036de 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -28,7 +28,7 @@ import static org.apache.hadoop.util.PlatformName.IBM_JAVA; import static org.apache.hadoop.util.StringUtils.getTrimmedStringCollection; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.io.File; import java.io.IOException; @@ -529,6 +529,14 @@ private void setLogin(LoginContext login) { user.setLogin(login); } + /** + * Set the last login time for logged in user + * @param loginTime the number of milliseconds since the beginning of time + */ + private void setLastLogin(long loginTime) { + user.setLastLogin(loginTime); + } + /** * Create a UserGroupInformation for the given subject. * This does not change the subject or acquire new credentials. @@ -1224,7 +1232,29 @@ public void reloginFromKeytab() throws IOException { reloginFromKeytab(false); } + /** + * Force re-Login a user in from a keytab file irrespective of the last login + * time. Loads a user identity from a keytab file and logs them in. They + * become the currently logged-in user. This method assumes that + * {@link #loginUserFromKeytab(String, String)} had happened already. The + * Subject field of this UserGroupInformation object is updated to have the + * new credentials. + * + * @throws IOException + * @throws KerberosAuthException on a failure + */ + @InterfaceAudience.Public + @InterfaceStability.Evolving + public void forceReloginFromKeytab() throws IOException { + reloginFromKeytab(false, true); + } + private void reloginFromKeytab(boolean checkTGT) throws IOException { + reloginFromKeytab(checkTGT, false); + } + + private void reloginFromKeytab(boolean checkTGT, boolean ignoreLastLoginTime) + throws IOException { if (!shouldRelogin() || !isFromKeytab()) { return; } @@ -1239,7 +1269,7 @@ private void reloginFromKeytab(boolean checkTGT) throws IOException { return; } } - relogin(login); + relogin(login, ignoreLastLoginTime); } /** @@ -1260,25 +1290,27 @@ public void reloginFromTicketCache() throws IOException { if (login == null) { throw new KerberosAuthException(MUST_FIRST_LOGIN); } - relogin(login); + relogin(login, false); } - private void relogin(HadoopLoginContext login) throws IOException { + private void relogin(HadoopLoginContext login, boolean ignoreLastLoginTime) + throws IOException { // ensure the relogin is atomic to avoid leaving credentials in an // inconsistent state. prevents other ugi instances, SASL, and SPNEGO // from accessing or altering credentials during the relogin. synchronized(login.getSubjectLock()) { // another racing thread may have beat us to the relogin. if (login == getLogin()) { - unprotectedRelogin(login); + unprotectedRelogin(login, ignoreLastLoginTime); } } } - private void unprotectedRelogin(HadoopLoginContext login) throws IOException { + private void unprotectedRelogin(HadoopLoginContext login, + boolean ignoreLastLoginTime) throws IOException { assert Thread.holdsLock(login.getSubjectLock()); long now = Time.now(); - if (!hasSufficientTimeElapsed(now)) { + if (!hasSufficientTimeElapsed(now) && !ignoreLastLoginTime) { return; } // register most recent relogin attempt @@ -1968,6 +2000,7 @@ private static UserGroupInformation doSubjectLogin( if (subject == null) { params.put(LoginParam.PRINCIPAL, ugi.getUserName()); ugi.setLogin(login); + ugi.setLastLogin(Time.now()); } return ugi; } catch (LoginException le) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java index df783f16edb90..260f1d22496f0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java @@ -24,7 +24,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.ProviderUtils; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -147,6 +147,10 @@ protected final String getPathAsString() { protected abstract String getSchemeName(); + protected abstract String getKeyStoreType(); + + protected abstract String getAlgorithm(); + protected abstract OutputStream getOutputStreamForKeystore() throws IOException; @@ -264,8 +268,8 @@ CredentialEntry innerSetCredential(String alias, char[] material) writeLock.lock(); try { keyStore.setKeyEntry(alias, - new SecretKeySpec(new String(material).getBytes("UTF-8"), "AES"), - password, null); + new SecretKeySpec(new String(material).getBytes("UTF-8"), + getAlgorithm()), password, null); } catch (KeyStoreException e) { throw new IOException("Can't store credential " + alias + " in " + this, e); @@ -315,7 +319,7 @@ private void locateKeystore() throws IOException { password = CREDENTIAL_PASSWORD_DEFAULT.toCharArray(); } KeyStore ks; - ks = KeyStore.getInstance("jceks"); + ks = KeyStore.getInstance(getKeyStoreType()); if (keystoreExists()) { stashOriginalFilePermissions(); try (InputStream in = getInputStreamForFile()) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/BouncyCastleFipsKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/BouncyCastleFipsKeyStoreProvider.java new file mode 100644 index 0000000000000..7c7c2c6cee561 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/BouncyCastleFipsKeyStoreProvider.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security.alias; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; +import java.net.URI; + +/** + * CredentialProvider based on Bouncy Castle FIPS KeyStore file format. + * The file may be stored in any Hadoop FileSystem using the following + * name mangling: + * bcfks://hdfs@nn1.example.com/my/creds.bcfks {@literal ->} + * hdfs://nn1.example.com/my/creds.bcfks bcfks://file/home/larry/creds.bcfks + * {@literal ->} file:///home/user1/creds.bcfks + */ +@InterfaceAudience.Private +public final class BouncyCastleFipsKeyStoreProvider extends KeyStoreProvider { + public static final String SCHEME_NAME = "bcfks"; + public static final String KEYSTORE_TYPE = "bcfks"; + public static final String ALGORITHM = "HMACSHA512"; + + private BouncyCastleFipsKeyStoreProvider(URI uri, Configuration conf) + throws IOException { + super(uri, conf); + } + + @Override + protected String getSchemeName() { + return SCHEME_NAME; + } + + @Override + protected String getKeyStoreType() { + return KEYSTORE_TYPE; + } + + @Override + protected String getAlgorithm() { + return ALGORITHM; + } + + /** + * The factory to create JksProviders, which is used by the ServiceLoader. + */ + public static class Factory extends CredentialProviderFactory { + @Override + public CredentialProvider createProvider(URI providerName, + Configuration conf) throws IOException { + if (SCHEME_NAME.equals(providerName.getScheme())) { + return new BouncyCastleFipsKeyStoreProvider(providerName, conf); + } + return null; + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java index 603772444bcef..f172f0828eb32 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java @@ -25,7 +25,7 @@ import java.util.Arrays; import java.util.List; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.StringUtils; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java index 5028482dfc4aa..f3b721f50b5a7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java @@ -20,14 +20,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.permission.FsPermission; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.net.URI; /** @@ -38,11 +32,10 @@ * {@literal ->} file:///home/larry/creds.jceks */ @InterfaceAudience.Private -public class JavaKeyStoreProvider extends AbstractJavaKeyStoreProvider { +public final class JavaKeyStoreProvider extends KeyStoreProvider { public static final String SCHEME_NAME = "jceks"; - - private FileSystem fs; - private FsPermission permissions; + public static final String KEYSTORE_TYPE = "jceks"; + public static final String ALGORITHM = "AES"; private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException { @@ -55,38 +48,13 @@ protected String getSchemeName() { } @Override - protected OutputStream getOutputStreamForKeystore() throws IOException { - FSDataOutputStream out = FileSystem.create(fs, getPath(), permissions); - return out; - } - - @Override - protected boolean keystoreExists() throws IOException { - return fs.exists(getPath()); - } - - @Override - protected InputStream getInputStreamForFile() throws IOException { - return fs.open(getPath()); + protected String getKeyStoreType() { + return KEYSTORE_TYPE; } @Override - protected void createPermissions(String perms) { - permissions = new FsPermission(perms); - } - - @Override - protected void stashOriginalFilePermissions() throws IOException { - // save off permissions in case we need to - // rewrite the keystore in flush() - FileStatus s = fs.getFileStatus(getPath()); - permissions = s.getPermission(); - } - - protected void initFileSystem(URI uri) - throws IOException { - super.initFileSystem(uri); - fs = getPath().getFileSystem(getConf()); + protected String getAlgorithm() { + return ALGORITHM; } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/KeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/KeyStoreProvider.java new file mode 100644 index 0000000000000..6909b07161a07 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/KeyStoreProvider.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security.alias; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.permission.FsPermission; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; + +/** + * CredentialProvider based on Java Key Store API. + * The file may be stored in any Hadoop FileSystem using the following + * name mangling: + * bcfks://hdfs@nn1.example.com/my/creds.bcfks {@literal ->} + * hdfs://nn1.example.com/my/creds.bcfks bcfks://file/home/larry/creds.bcfks + * {@literal ->} file:///home/user1/creds.bcfks + */ +@InterfaceAudience.Private +public abstract class KeyStoreProvider extends AbstractJavaKeyStoreProvider { + + private FileSystem fs; + private FsPermission permissions; + + protected KeyStoreProvider(URI uri, Configuration conf) + throws IOException { + super(uri, conf); + } + + @Override + protected OutputStream getOutputStreamForKeystore() throws IOException { + FSDataOutputStream out = FileSystem.create(fs, getPath(), permissions); + return out; + } + + @Override + protected boolean keystoreExists() throws IOException { + return fs.exists(getPath()); + } + + @Override + protected InputStream getInputStreamForFile() throws IOException { + return fs.open(getPath()); + } + + @Override + protected void createPermissions(String perms) { + permissions = new FsPermission(perms); + } + + @Override + protected void stashOriginalFilePermissions() throws IOException { + // save off permissions in case we need to + // rewrite the keystore in flush() + FileStatus s = fs.getFileStatus(getPath()); + permissions = s.getPermission(); + } + + protected void initFileSystem(URI uri) + throws IOException { + super.initFileSystem(uri); + fs = getPath().getFileSystem(getConf()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalBouncyCastleFipsKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalBouncyCastleFipsKeyStoreProvider.java new file mode 100644 index 0000000000000..1aef63a90db8e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalBouncyCastleFipsKeyStoreProvider.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security.alias; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; +import java.net.URI; + +/** + * CredentialProvider based on bouncy castle FIPS KeyStore file format. + * The file may be stored only on the local filesystem using the + * following name mangling: + * localbcfks://file/home/larry/creds.bcfks {@literal ->} + * file:///home/larry/creds.bcfks + */ +@InterfaceAudience.Private +public final class LocalBouncyCastleFipsKeyStoreProvider extends + LocalKeyStoreProvider { + public static final String SCHEME_NAME = "localbcfks"; + public static final String KEYSTORE_TYPE = "bcfks"; + public static final String ALGORITHM = "HMACSHA512"; + + private LocalBouncyCastleFipsKeyStoreProvider(URI uri, Configuration conf) + throws IOException { + super(uri, conf); + } + + @Override + protected String getSchemeName() { + return SCHEME_NAME; + } + + @Override + protected String getKeyStoreType() { + return KEYSTORE_TYPE; + } + + @Override + protected String getAlgorithm() { + return ALGORITHM; + } + + /** + * The factory to create KeyStore Providers, which is used by the + * ServiceLoader. + */ + public static class Factory extends CredentialProviderFactory { + @Override + public CredentialProvider createProvider(URI providerName, + Configuration conf) throws IOException { + if (SCHEME_NAME.equals(providerName.getScheme())) { + return new LocalBouncyCastleFipsKeyStoreProvider(providerName, conf); + } + return null; + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java index c44e246b9d0cd..dd922412892df 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java @@ -20,24 +20,9 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.util.Shell; -import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.Set; -import java.util.StringTokenizer; -import java.util.EnumSet; /** * CredentialProvider based on Java's KeyStore file format. The file may be @@ -47,10 +32,10 @@ */ @InterfaceAudience.Private public final class LocalJavaKeyStoreProvider extends - AbstractJavaKeyStoreProvider { + LocalKeyStoreProvider { public static final String SCHEME_NAME = "localjceks"; - private File file; - private Set permissions; + public static final String KEYSTORE_TYPE = "jceks"; + public static final String ALGORITHM = "AES"; private LocalJavaKeyStoreProvider(URI uri, Configuration conf) throws IOException { @@ -63,106 +48,13 @@ protected String getSchemeName() { } @Override - protected OutputStream getOutputStreamForKeystore() throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("using '" + file + "' for output stream."); - } - OutputStream out = Files.newOutputStream(file.toPath()); - return out; - } - - @Override - protected boolean keystoreExists() throws IOException { - /* The keystore loader doesn't handle zero length files. */ - return file.exists() && (file.length() > 0); - } - - @Override - protected InputStream getInputStreamForFile() throws IOException { - InputStream is = Files.newInputStream(file.toPath()); - return is; - } - - @Override - protected void createPermissions(String perms) throws IOException { - int mode = 700; - try { - mode = Integer.parseInt(perms, 8); - } catch (NumberFormatException nfe) { - throw new IOException("Invalid permissions mode provided while " - + "trying to createPermissions", nfe); - } - permissions = modeToPosixFilePermission(mode); - } - - @Override - protected void stashOriginalFilePermissions() throws IOException { - // save off permissions in case we need to - // rewrite the keystore in flush() - if (!Shell.WINDOWS) { - Path path = Paths.get(file.getCanonicalPath()); - permissions = Files.getPosixFilePermissions(path); - } else { - // On Windows, the JDK does not support the POSIX file permission APIs. - // Instead, we can do a winutils call and translate. - String[] cmd = Shell.getGetPermissionCommand(); - String[] args = new String[cmd.length + 1]; - System.arraycopy(cmd, 0, args, 0, cmd.length); - args[cmd.length] = file.getCanonicalPath(); - String out = Shell.execCommand(args); - StringTokenizer t = new StringTokenizer(out, Shell.TOKEN_SEPARATOR_REGEX); - // The winutils output consists of 10 characters because of the leading - // directory indicator, i.e. "drwx------". The JDK parsing method expects - // a 9-character string, so remove the leading character. - String permString = t.nextToken().substring(1); - permissions = PosixFilePermissions.fromString(permString); - } - } - - @Override - protected void initFileSystem(URI uri) - throws IOException { - super.initFileSystem(uri); - try { - file = new File(new URI(getPath().toString())); - if (LOG.isDebugEnabled()) { - LOG.debug("initialized local file as '" + file + "'."); - if (file.exists()) { - LOG.debug("the local file exists and is size " + file.length()); - if (LOG.isTraceEnabled()) { - if (file.canRead()) { - LOG.trace("we can read the local file."); - } - if (file.canWrite()) { - LOG.trace("we can write the local file."); - } - } - } else { - LOG.debug("the local file does not exist."); - } - } - } catch (URISyntaxException e) { - throw new IOException(e); - } + protected String getKeyStoreType() { + return KEYSTORE_TYPE; } @Override - public void flush() throws IOException { - super.flush(); - if (LOG.isDebugEnabled()) { - LOG.debug("Resetting permissions to '" + permissions + "'"); - } - if (!Shell.WINDOWS) { - Files.setPosixFilePermissions(Paths.get(file.getCanonicalPath()), - permissions); - } else { - // FsPermission expects a 10-character string because of the leading - // directory indicator, i.e. "drwx------". The JDK toString method returns - // a 9-character string, so prepend a leading character. - FsPermission fsPermission = FsPermission.valueOf( - "-" + PosixFilePermissions.toString(permissions)); - FileUtil.setPermission(file, fsPermission); - } + protected String getAlgorithm() { + return ALGORITHM; } /** @@ -178,37 +70,4 @@ public CredentialProvider createProvider(URI providerName, return null; } } - - private static Set modeToPosixFilePermission( - int mode) { - Set perms = EnumSet.noneOf(PosixFilePermission.class); - if ((mode & 0001) != 0) { - perms.add(PosixFilePermission.OTHERS_EXECUTE); - } - if ((mode & 0002) != 0) { - perms.add(PosixFilePermission.OTHERS_WRITE); - } - if ((mode & 0004) != 0) { - perms.add(PosixFilePermission.OTHERS_READ); - } - if ((mode & 0010) != 0) { - perms.add(PosixFilePermission.GROUP_EXECUTE); - } - if ((mode & 0020) != 0) { - perms.add(PosixFilePermission.GROUP_WRITE); - } - if ((mode & 0040) != 0) { - perms.add(PosixFilePermission.GROUP_READ); - } - if ((mode & 0100) != 0) { - perms.add(PosixFilePermission.OWNER_EXECUTE); - } - if ((mode & 0200) != 0) { - perms.add(PosixFilePermission.OWNER_WRITE); - } - if ((mode & 0400) != 0) { - perms.add(PosixFilePermission.OWNER_READ); - } - return perms; - } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalKeyStoreProvider.java new file mode 100644 index 0000000000000..b355bbc9cd62f --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalKeyStoreProvider.java @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security.alias; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Shell; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.EnumSet; +import java.util.Set; +import java.util.StringTokenizer; + +/** + * CredentialProvider based on Java's KeyStore file format. The file may be + * stored only on the local filesystem using the following name mangling: + * localjceks://file/home/larry/creds.jceks {@literal ->} + * file:///home/larry/creds.jceks + */ +@InterfaceAudience.Private +public abstract class LocalKeyStoreProvider extends + AbstractJavaKeyStoreProvider { + private File file; + private Set permissions; + + protected LocalKeyStoreProvider(URI uri, Configuration conf) + throws IOException { + super(uri, conf); + } + + @Override + protected OutputStream getOutputStreamForKeystore() throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("using '" + file + "' for output stream."); + } + OutputStream out = Files.newOutputStream(file.toPath()); + return out; + } + + @Override + protected boolean keystoreExists() throws IOException { + /* The keystore loader doesn't handle zero length files. */ + return file.exists() && (file.length() > 0); + } + + @Override + protected InputStream getInputStreamForFile() throws IOException { + InputStream is = Files.newInputStream(file.toPath()); + return is; + } + + @Override + protected void createPermissions(String perms) throws IOException { + int mode = 700; + try { + mode = Integer.parseInt(perms, 8); + } catch (NumberFormatException nfe) { + throw new IOException("Invalid permissions mode provided while " + + "trying to createPermissions", nfe); + } + permissions = modeToPosixFilePermission(mode); + } + + @Override + protected void stashOriginalFilePermissions() throws IOException { + // save off permissions in case we need to + // rewrite the keystore in flush() + if (!Shell.WINDOWS) { + Path path = Paths.get(file.getCanonicalPath()); + permissions = Files.getPosixFilePermissions(path); + } else { + // On Windows, the JDK does not support the POSIX file permission APIs. + // Instead, we can do a winutils call and translate. + String[] cmd = Shell.getGetPermissionCommand(); + String[] args = new String[cmd.length + 1]; + System.arraycopy(cmd, 0, args, 0, cmd.length); + args[cmd.length] = file.getCanonicalPath(); + String out = Shell.execCommand(args); + StringTokenizer t = new StringTokenizer(out, Shell.TOKEN_SEPARATOR_REGEX); + // The winutils output consists of 10 characters because of the leading + // directory indicator, i.e. "drwx------". The JDK parsing method expects + // a 9-character string, so remove the leading character. + String permString = t.nextToken().substring(1); + permissions = PosixFilePermissions.fromString(permString); + } + } + + @Override + protected void initFileSystem(URI uri) + throws IOException { + super.initFileSystem(uri); + try { + file = new File(new URI(getPath().toString())); + if (LOG.isDebugEnabled()) { + LOG.debug("initialized local file as '" + file + "'."); + if (file.exists()) { + LOG.debug("the local file exists and is size " + file.length()); + if (LOG.isTraceEnabled()) { + if (file.canRead()) { + LOG.trace("we can read the local file."); + } + if (file.canWrite()) { + LOG.trace("we can write the local file."); + } + } + } else { + LOG.debug("the local file does not exist."); + } + } + } catch (URISyntaxException e) { + throw new IOException(e); + } + } + + @Override + public void flush() throws IOException { + super.flush(); + if (LOG.isDebugEnabled()) { + LOG.debug("Resetting permissions to '" + permissions + "'"); + } + if (!Shell.WINDOWS) { + Files.setPosixFilePermissions(Paths.get(file.getCanonicalPath()), + permissions); + } else { + // FsPermission expects a 10-character string because of the leading + // directory indicator, i.e. "drwx------". The JDK toString method returns + // a 9-character string, so prepend a leading character. + FsPermission fsPermission = FsPermission.valueOf( + "-" + PosixFilePermissions.toString(permissions)); + FileUtil.setPermission(file, fsPermission); + } + } + + private static Set modeToPosixFilePermission( + int mode) { + Set perms = EnumSet.noneOf(PosixFilePermission.class); + if ((mode & 0001) != 0) { + perms.add(PosixFilePermission.OTHERS_EXECUTE); + } + if ((mode & 0002) != 0) { + perms.add(PosixFilePermission.OTHERS_WRITE); + } + if ((mode & 0004) != 0) { + perms.add(PosixFilePermission.OTHERS_READ); + } + if ((mode & 0010) != 0) { + perms.add(PosixFilePermission.GROUP_EXECUTE); + } + if ((mode & 0020) != 0) { + perms.add(PosixFilePermission.GROUP_WRITE); + } + if ((mode & 0040) != 0) { + perms.add(PosixFilePermission.GROUP_READ); + } + if ((mode & 0100) != 0) { + perms.add(PosixFilePermission.OWNER_EXECUTE); + } + if ((mode & 0200) != 0) { + perms.add(PosixFilePermission.OWNER_WRITE); + } + if ((mode & 0400) != 0) { + perms.add(PosixFilePermission.OWNER_READ); + } + return perms; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/package-info.java new file mode 100644 index 0000000000000..d05e3cb9f20f3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provides the hadoop credential provider API. + */ +package org.apache.hadoop.security.alias; \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java index b766d5c37fa2f..9bb7bcc988330 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java @@ -30,7 +30,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.MachineList; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; @InterfaceStability.Unstable @InterfaceAudience.Public diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java index 97a7f080fbdf9..2adf77bbe8d18 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java @@ -18,7 +18,7 @@ package org.apache.hadoop.security.authorize; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -26,7 +26,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; @InterfaceStability.Unstable @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce", "HBase", "Hive"}) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java index a264eb4dcd9fb..27a6355b6531c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java @@ -33,7 +33,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.MachineList; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java index 60c2864bbe539..6ba651c13da0b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java @@ -36,7 +36,7 @@ import org.apache.commons.lang3.StringUtils; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/DelegatingSSLSocketFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/DelegatingSSLSocketFactory.java index ff650d6c392db..5644234a57a7f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/DelegatingSSLSocketFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/DelegatingSSLSocketFactory.java @@ -30,7 +30,7 @@ import javax.net.ssl.SSLSocket; import javax.net.ssl.SSLSocketFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,19 +43,20 @@ * *

* The factory has several different modes of operation: - *

    - *
  • OpenSSL: Uses the wildly-openssl library to delegate to the - * system installed OpenSSL. If the wildfly-openssl integration is not - * properly setup, an exception is thrown.
    • - *
  • Default: Attempts to use the OpenSSL mode, if it cannot load the - * necessary libraries, it falls back to the Default_JSEE mode.
    • - *
  • Default_JSSE: Delegates to the JSSE implementation of SSL, but - * it disables the GCM cipher when running on Java 8.
    • - *
  • Default_JSSE_with_GCM: Delegates to the JSSE implementation of - * SSL with no modification to the list of enabled ciphers.
    • - *
*

* + *
    + *
  • OpenSSL: Uses the wildly-openssl library to delegate to the + * system installed OpenSSL. If the wildfly-openssl integration is not + * properly setup, an exception is thrown.
    • + *
  • Default: Attempts to use the OpenSSL mode, if it cannot load the + * necessary libraries, it falls back to the Default_JSEE mode.
    • + *
  • Default_JSSE: Delegates to the JSSE implementation of SSL, but + * it disables the GCM cipher when running on Java 8.
    • + *
  • Default_JSSE_with_GCM: Delegates to the JSSE implementation of + * SSL with no modification to the list of enabled ciphers.
    • + *
+ * * In order to load OpenSSL, applications must ensure the wildfly-openssl * artifact is on the classpath. Currently, only ABFS declares * wildfly-openssl as an explicit dependency. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java index 3531173bb72f7..b184e4a152b8b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.security.ssl; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java index b2f0118aaf5c6..7430477932292 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java @@ -21,7 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java index e10741e58a5ce..d168a317dfd7e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java @@ -102,7 +102,7 @@ public enum Mode { CLIENT, SERVER } public static final String SSLCERTIFICATE = IBM_JAVA?"ibmX509":"SunX509"; public static final String KEYSTORES_FACTORY_CLASS_KEY = - "hadoop.ssl.keystores.factory.class"; + "hadoop.ssl.keystores.factory.class"; private Configuration conf; private Mode mode; @@ -165,6 +165,13 @@ public static Configuration readSSLConfiguration(Configuration conf, SSL_SERVER_CONF_DEFAULT); } sslConf.addResource(sslConfResource); + // Only fallback to input config if classpath SSL config does not load for + // backward compatibility. + if (sslConf.getResource(sslConfResource) == null) { + LOG.debug("{} can't be loaded form classpath, fallback using SSL" + + " config from input configuration.", sslConfResource); + sslConf = conf; + } return sslConf; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java index 4f0f6fc4d444a..e521a7c07b92f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java @@ -18,8 +18,8 @@ package org.apache.hadoop.security.token; -import com.google.common.collect.Maps; -import com.google.common.primitives.Bytes; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Bytes; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.HadoopIllegalArgumentException; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java index 6dfe52a83bf68..3f27e45af8191 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java @@ -32,7 +32,7 @@ import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.TokenIdentifier; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; @InterfaceAudience.Public @InterfaceStability.Evolving @@ -49,7 +49,7 @@ public abstract class AbstractDelegationTokenIdentifier private int masterKeyId = 0; public AbstractDelegationTokenIdentifier() { - this(new Text(), new Text(), new Text()); + this(null, null, null); } public AbstractDelegationTokenIdentifier(Text owner, Text renewer, Text realUser) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java index eb65799757f66..dacdf4469555e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java @@ -46,7 +46,7 @@ import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.Time; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java index 276573ba00c9b..1de786ebdc783 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java @@ -66,8 +66,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * An implementation of {@link AbstractDelegationTokenSecretManager} that diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java index 4e9881bc34369..07507f9dab212 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.security.token.delegation.web; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.Text; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java index 5275526202f2b..ae42c7c458ea2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.security.token.delegation.web; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.curator.framework.CuratorFramework; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java index 1de534f36ba4f..d4499aea0ca04 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java @@ -53,7 +53,7 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * An {@link AuthenticationHandler} that implements Kerberos SPNEGO mechanism diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java index e1445fb5ca05b..7e7e794d9de47 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java @@ -33,7 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Delegation Token Manager used by the diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java index 0661fb2b5a2a7..865977e67d07a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java @@ -36,8 +36,8 @@ import org.apache.hadoop.security.authentication.server.HttpConstants; import org.apache.hadoop.security.authentication.server.MultiSchemeAuthenticationHandler; -import com.google.common.base.Preconditions; -import com.google.common.base.Splitter; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Splitter; /** * A {@link CompositeAuthenticationHandler} that supports multiple HTTP diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/AbstractService.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/AbstractService.java index 0aa5bafe88423..bc76e642b6027 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/AbstractService.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/AbstractService.java @@ -30,7 +30,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/InterruptEscalator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/InterruptEscalator.java index 594fc5bfe8d14..ad92d4c6d7a24 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/InterruptEscalator.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/InterruptEscalator.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java index 17aa9639c31d9..bcb589f24885f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java @@ -20,7 +20,7 @@ import java.util.concurrent.atomic.AtomicInteger; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import sun.misc.Signal; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java index 6d161bf8b613c..47aabed89cdf0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java @@ -26,8 +26,8 @@ import java.util.Collections; import java.util.List; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/AutoCloseableLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/AutoCloseableLock.java index d7fe93d73cf02..e761858e3c170 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/AutoCloseableLock.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/AutoCloseableLock.java @@ -21,7 +21,7 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * This is a wrap class of a ReentrantLock. Extending AutoCloseable diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BlockingThreadPoolExecutorService.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BlockingThreadPoolExecutorService.java index d49013ec14d1f..451b5f5d6ce1e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BlockingThreadPoolExecutorService.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BlockingThreadPoolExecutorService.java @@ -28,7 +28,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.util.concurrent.MoreExecutors; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ChunkedArrayList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ChunkedArrayList.java index 84ddc32f88c1f..ff7197ce52e4d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ChunkedArrayList.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ChunkedArrayList.java @@ -23,10 +23,10 @@ import org.apache.hadoop.classification.InterfaceAudience; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * Simplified List implementation which stores elements as a list diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CloseableReferenceCount.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CloseableReferenceCount.java index 388a087bbd346..f81a429b5d422 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CloseableReferenceCount.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CloseableReferenceCount.java @@ -21,7 +21,7 @@ import java.nio.channels.ClosedChannelException; import java.util.concurrent.atomic.AtomicInteger; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * A closeable object that maintains a reference count. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DirectBufferPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DirectBufferPool.java index 510938b7fff95..9f4cc1921a353 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DirectBufferPool.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DirectBufferPool.java @@ -26,7 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceStability; /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java index 2ee53dc595f99..ee70256590bc4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java @@ -25,7 +25,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.FileUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java index 7d04db23ca76c..1e5e9644beee7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.util; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.util.DiskChecker.DiskErrorException; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java index 690d09755171f..846af7fb74372 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java @@ -17,7 +17,7 @@ package org.apache.hadoop.util; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.util.StringUtils; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java index 4247eb7050b5a..e8dc8cb447092 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java @@ -18,7 +18,7 @@ package org.apache.hadoop.util; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import java.lang.management.GarbageCollectorMXBean; import java.lang.management.ManagementFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdentityHashStore.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdentityHashStore.java index 3ae4bbac6591d..81a22a46aa850 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdentityHashStore.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdentityHashStore.java @@ -21,7 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * The IdentityHashStore stores (key, value) mappings in an array. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedLock.java index cc0ebdf8b3e39..a5f4f0709dd73 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedLock.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedLock.java @@ -26,7 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java index 8ab392ed041d0..8417246f0467c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedWriteLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedWriteLock.java index 6842166930d5f..710861c761ae3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedWriteLock.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedWriteLock.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IntrusiveCollection.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IntrusiveCollection.java index 1ffb7db3febff..126aa6ab00884 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IntrusiveCollection.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IntrusiveCollection.java @@ -23,7 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java index e043b1dc382c0..7a26ecadee456 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java @@ -35,7 +35,7 @@ import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java index 420ac8bc1851e..9c9953fb14c6d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java @@ -28,11 +28,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.AbstractService; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java index 79de1ac554476..01ddf373c085e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java @@ -24,8 +24,8 @@ import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * A low memory footprint Cache which extends {@link LightWeightGSet}. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightGSet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightGSet.java index 7c7878a71bdf2..d2ea5cf350c66 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightGSet.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightGSet.java @@ -27,7 +27,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * A low memory footprint {@link GSet} implementation, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LimitInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LimitInputStream.java index bd646e0bcb608..de95e98e1f4dd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LimitInputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LimitInputStream.java @@ -19,8 +19,8 @@ package org.apache.hadoop.util; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull; import java.io.FilterInputStream; import java.io.IOException; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java index 20931deb0e9bd..c52fb94038606 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java @@ -28,8 +28,8 @@ import org.apache.commons.net.util.SubnetUtils; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.net.InetAddresses; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java index a8a380ed070d1..11d1176f92a59 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java @@ -74,11 +74,6 @@ public static boolean isNativeCodeLoaded() { return nativeCodeLoaded; } - /** - * Returns true only if this build was compiled with support for snappy. - */ - public static native boolean buildSupportsSnappy(); - /** * Returns true only if this build was compiled with support for ISA-L. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java index 3142df2da36e0..cc41f02d87e48 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java @@ -21,7 +21,7 @@ import org.apache.hadoop.fs.ChecksumException; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Wrapper around JNI support code to do checksum computation diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java index 23388248575ac..e40f01195ba07 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java @@ -23,7 +23,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.OpensslCipher; import org.apache.hadoop.io.compress.Lz4Codec; -import org.apache.hadoop.io.compress.SnappyCodec; import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; import org.apache.hadoop.io.compress.zlib.ZlibFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -67,7 +66,6 @@ public static void main(String[] args) { Configuration conf = new Configuration(); boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded(); boolean zlibLoaded = false; - boolean snappyLoaded = false; boolean isalLoaded = false; boolean zStdLoaded = false; boolean pmdkLoaded = false; @@ -80,7 +78,6 @@ public static void main(String[] args) { String openSslDetail = ""; String hadoopLibraryName = ""; String zlibLibraryName = ""; - String snappyLibraryName = ""; String isalDetail = ""; String pmdkDetail = ""; String zstdLibraryName = ""; @@ -99,11 +96,6 @@ public static void main(String[] args) { if (zStdLoaded && NativeCodeLoader.buildSupportsZstd()) { zstdLibraryName = ZStandardCodec.getLibraryName(); } - snappyLoaded = NativeCodeLoader.buildSupportsSnappy() && - SnappyCodec.isNativeCodeLoaded(); - if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) { - snappyLibraryName = SnappyCodec.getLibraryName(); - } isalDetail = ErasureCodeNative.getLoadingFailureReason(); if (isalDetail != null) { @@ -152,7 +144,6 @@ public static void main(String[] args) { System.out.printf("hadoop: %b %s%n", nativeHadoopLoaded, hadoopLibraryName); System.out.printf("zlib: %b %s%n", zlibLoaded, zlibLibraryName); System.out.printf("zstd : %b %s%n", zStdLoaded, zstdLibraryName); - System.out.printf("snappy: %b %s%n", snappyLoaded, snappyLibraryName); System.out.printf("lz4: %b %s%n", lz4Loaded, lz4LibraryName); System.out.printf("bzip2: %b %s%n", bzip2Loaded, bzip2LibraryName); System.out.printf("openssl: %b %s%n", openSslLoaded, openSslDetail); @@ -164,7 +155,7 @@ public static void main(String[] args) { } if ((!nativeHadoopLoaded) || (Shell.WINDOWS && (!winutilsExists)) || - (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded + (checkAll && !(zlibLoaded && lz4Loaded && bzip2Loaded && isalLoaded && zStdLoaded))) { // return 1 to indicated check failed ExitUtil.terminate(1); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReadWriteDiskValidatorMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReadWriteDiskValidatorMetrics.java index 620186298de66..83cc6dcd9b576 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReadWriteDiskValidatorMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReadWriteDiskValidatorMetrics.java @@ -18,7 +18,7 @@ package org.apache.hadoop.util; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsSystem; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java index 1ae71d187d3ba..31b0546d36517 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java @@ -124,15 +124,35 @@ private static void setJobConf(Object theObject, Configuration conf) { */ @SuppressWarnings("unchecked") public static T newInstance(Class theClass, Configuration conf) { + return newInstance(theClass, conf, EMPTY_ARRAY); + } + + /** Create an object for the given class and initialize it from conf + * + * @param theClass class of which an object is created + * @param conf Configuration + * @param argTypes the types of the arguments + * @param values the values of the arguments + * @return a new object + */ + @SuppressWarnings("unchecked") + public static T newInstance(Class theClass, Configuration conf, + Class[] argTypes, Object ... values) { T result; + if (argTypes.length != values.length) { + throw new IllegalArgumentException(argTypes.length + + " parameters are required but " + + values.length + + " arguments are provided"); + } try { Constructor meth = (Constructor) CONSTRUCTOR_CACHE.get(theClass); if (meth == null) { - meth = theClass.getDeclaredConstructor(EMPTY_ARRAY); + meth = theClass.getDeclaredConstructor(argTypes); meth.setAccessible(true); CONSTRUCTOR_CACHE.put(theClass, meth); } - result = meth.newInstance(); + result = meth.newInstance(values); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SemaphoredDelegatingExecutor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SemaphoredDelegatingExecutor.java index 4ec77e75ba520..1f29ba8b5e3a4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SemaphoredDelegatingExecutor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SemaphoredDelegatingExecutor.java @@ -18,10 +18,10 @@ package org.apache.hadoop.util; -import com.google.common.util.concurrent.ForwardingListeningExecutorService; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ForwardingListeningExecutorService; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java index 9ba9e94dff5aa..5096b10951d38 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java @@ -26,7 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; @InterfaceAudience.Private @InterfaceStability.Unstable diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index 0dc49739c4b5a..96299b6c31cfa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -35,7 +35,7 @@ import java.util.WeakHashMap; import java.util.concurrent.atomic.AtomicBoolean; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java index 76d90063609b2..1b43c5d0ac85e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.util; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java index 50a728e568a4e..296727e67b44f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java @@ -18,7 +18,7 @@ package org.apache.hadoop.util; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java index 028e49acbe59a..4a30ee2800aa9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java @@ -21,8 +21,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import com.google.common.collect.Interner; -import com.google.common.collect.Interners; +import org.apache.hadoop.thirdparty.com.google.common.collect.Interner; +import org.apache.hadoop.thirdparty.com.google.common.collect.Interners; /** * Provides string interning utility methods. For weak interning, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java index cf7b04ab61a7e..9e7b36f71e211 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java @@ -42,8 +42,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.net.NetUtils; -import com.google.common.base.Preconditions; -import com.google.common.net.InetAddresses; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses; /** * General string utils diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java index 3591bb81e121c..6f2f585c87f22 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java @@ -30,7 +30,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java index e8940141b43c5..4d86153345bae 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java @@ -19,7 +19,7 @@ import java.io.IOException; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java index 48cef5f06fcc9..8e4e67d1b61e0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java @@ -27,10 +27,10 @@ import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Id; -import com.google.common.base.Charsets; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; -import com.google.common.io.Files; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Splitter; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.io.Files; /** * Utilities for working with ZooKeeper. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGetFuture.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGetFuture.java index 61eb777d4806d..46fc8df37d3f3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGetFuture.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGetFuture.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.util.concurrent; -import com.google.common.util.concurrent.AbstractFuture; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.AbstractFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ChildReaper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ChildReaper.java index 86142fb6d3a4f..e125dbfbd0abb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ChildReaper.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ChildReaper.java @@ -18,7 +18,7 @@ */ package org.apache.hadoop.util.curator; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.curator.framework.recipes.locks.Reaper; import org.apache.curator.utils.CloseableUtils; import org.apache.curator.framework.CuratorFramework; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java index 36dade27dd6f7..3e3939515bc1d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java @@ -39,7 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Helper class that provides utility methods specific to ZK operations. diff --git a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj index ac3767b276882..19b4d95e43622 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj +++ b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj @@ -17,7 +17,7 @@ limitations under the License. --> - + Release @@ -69,15 +69,6 @@ hadoop - $(CustomSnappyPrefix) - $(CustomSnappyPrefix)\lib - $(CustomSnappyPrefix)\bin - $(CustomSnappyLib) - $(CustomSnappyPrefix) - $(CustomSnappyPrefix)\include - $(CustomSnappyInclude) - true - $(SnappyInclude);$(IncludePath) $(ZLIB_HOME);$(IncludePath) @@ -87,11 +78,6 @@ $(CustomIsalLib) true - - - - - /D HADOOP_SNAPPY_LIBRARY=L\"snappy.dll\" - - - /D HADOOP_SNAPPY_LIBRARY=L\"snappy.dll\" - @@ -157,7 +137,6 @@ - /D HADOOP_SNAPPY_LIBRARY=L\"snappy.dll\" /D HADOOP_ISAL_LIBRARY=L\"isa-l.dll\" @@ -181,7 +160,6 @@ - diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c index abff7ea5f17ff..f60a19a662c4c 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c @@ -46,6 +46,13 @@ static int (*dlsym_EVP_CipherUpdate)(EVP_CIPHER_CTX *, unsigned char *, \ static int (*dlsym_EVP_CipherFinal_ex)(EVP_CIPHER_CTX *, unsigned char *, int *); static EVP_CIPHER * (*dlsym_EVP_aes_256_ctr)(void); static EVP_CIPHER * (*dlsym_EVP_aes_128_ctr)(void); +#if OPENSSL_VERSION_NUMBER >= 0x10100001L +static EVP_CIPHER * (*dlsym_EVP_sm4_ctr)(void); +static int (*dlsym_OPENSSL_init_crypto)(uint64_t opts, \ + const OPENSSL_INIT_SETTINGS *settings); +static ENGINE * (*dlsym_ENGINE_by_id)(const char *id); +static int (*dlsym_ENGINE_free)(ENGINE *); +#endif static void *openssl; #endif @@ -84,6 +91,18 @@ static __dlsym_EVP_CipherUpdate dlsym_EVP_CipherUpdate; static __dlsym_EVP_CipherFinal_ex dlsym_EVP_CipherFinal_ex; static __dlsym_EVP_aes_256_ctr dlsym_EVP_aes_256_ctr; static __dlsym_EVP_aes_128_ctr dlsym_EVP_aes_128_ctr; +#if OPENSSL_VERSION_NUMBER >= 0x10101001L +typedef EVP_CIPHER * (__cdecl *__dlsym_EVP_sm4_ctr)(void); +typedef int (__cdecl *__dlsym_OPENSSL_init_crypto)(uint64_t opts, \ + const OPENSSL_INIT_SETTINGS *settings); +typedef ENGINE * (__cdecl *__dlsym_ENGINE_by_id)(const char *id); +typedef int (__cdecl *__dlsym_ENGINE_free)(ENGINE *e); + +static __dlsym_EVP_sm4_ctr dlsym_EVP_sm4_ctr; +static __dlsym_OPENSSL_init_crypto dlsym_OPENSSL_init_crypto; +static __dlsym_ENGINE_by_id dlsym_ENGINE_by_id; +static __dlsym_ENGINE_free dlsym_ENGINE_free; +#endif static HMODULE openssl; #endif @@ -102,6 +121,15 @@ static void loadAesCtr(JNIEnv *env) #endif } +static void loadSm4Ctr(JNIEnv *env) +{ +#ifdef UNIX +#if OPENSSL_VERSION_NUMBER >= 0x10101001L + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_sm4_ctr, env, openssl, "EVP_sm4_ctr"); +#endif +#endif +} + JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs (JNIEnv *env, jclass clazz) { @@ -153,6 +181,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs "EVP_CipherUpdate"); LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherFinal_ex, env, openssl, \ "EVP_CipherFinal_ex"); +#if OPENSSL_VERSION_NUMBER >= 0x10101001L + LOAD_DYNAMIC_SYMBOL(dlsym_OPENSSL_init_crypto, env, openssl, \ + "OPENSSL_init_crypto"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, \ + "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, \ + "ENGINE_free"); +#endif #endif #ifdef WINDOWS @@ -185,14 +221,31 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs env, openssl, "EVP_CipherUpdate"); LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherFinal_ex, dlsym_EVP_CipherFinal_ex, \ env, openssl, "EVP_CipherFinal_ex"); +#if OPENSSL_VERSION_NUMBER >= 0x10101001L + LOAD_DYNAMIC_SYMBOL(__dlsym_OPENSSL_init_crypto, dlsym_OPENSSL_init_crypto, \ + env, openssl, "OPENSSL_init_crypto"); + LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_by_id, dlsym_ENGINE_by_id, \ + env, openssl, "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_free, dlsym_ENGINE_free, \ + env, openssl, "ENGINE_by_free"); +#endif #endif loadAesCtr(env); + loadSm4Ctr(env); +#if OPENSSL_VERSION_NUMBER >= 0x10101001L + int ret = dlsym_OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); + if(!ret) { + THROW(env, "java/lang/UnsatisfiedLinkError", \ + "Openssl init crypto failed"); + return; + } +#endif jthrowable jthr = (*env)->ExceptionOccurred(env); if (jthr) { (*env)->DeleteLocalRef(env, jthr); THROW(env, "java/lang/UnsatisfiedLinkError", \ - "Cannot find AES-CTR support, is your version of Openssl new enough?"); + "Cannot find AES-CTR/SM4-CTR support, is your version of Openssl new enough?"); return; } } @@ -200,7 +253,7 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initContext (JNIEnv *env, jclass clazz, jint alg, jint padding) { - if (alg != AES_CTR) { + if (alg != AES_CTR && alg != SM4_CTR) { THROW(env, "java/security/NoSuchAlgorithmException", NULL); return (jlong)0; } @@ -209,11 +262,27 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initContext return (jlong)0; } - if (dlsym_EVP_aes_256_ctr == NULL || dlsym_EVP_aes_128_ctr == NULL) { + if (alg == AES_CTR && (dlsym_EVP_aes_256_ctr == NULL || dlsym_EVP_aes_128_ctr == NULL)) { THROW(env, "java/security/NoSuchAlgorithmException", \ "Doesn't support AES CTR."); return (jlong)0; } + + if (alg == SM4_CTR) { + int ret = 0; +#if OPENSSL_VERSION_NUMBER >= 0x10101001L + if (dlsym_EVP_sm4_ctr == NULL) { + ret = 1; + } +#else + ret = 1; +#endif + if (ret) { + THROW(env, "java/security/NoSuchAlgorithmException", \ + "Doesn't support SM4 CTR."); + return (jlong)0; + } + } // Create and initialize a EVP_CIPHER_CTX EVP_CIPHER_CTX *context = dlsym_EVP_CIPHER_CTX_new(); @@ -225,7 +294,29 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initContext return JLONG(context); } -// Only supports AES-CTR currently +JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initEngine + (JNIEnv *env, jclass clazz, jstring engineId) +{ + ENGINE *e = NULL; + +#if OPENSSL_VERSION_NUMBER >= 0x10101001L + if (engineId != NULL) { + const char *id = (*env)->GetStringUTFChars(env, engineId, NULL); + if (id != NULL) { + e = dlsym_ENGINE_by_id(id); + (*env)->ReleaseStringUTFChars(env, engineId, id); + } + } +#endif + + if (e == NULL) { + return (jlong)0; + } else { + return JLONG(e); + } +} + +// Only supports AES-CTR & SM4-CTR currently static EVP_CIPHER * getEvpCipher(int alg, int keyLen) { EVP_CIPHER *cipher = NULL; @@ -235,13 +326,19 @@ static EVP_CIPHER * getEvpCipher(int alg, int keyLen) } else if (keyLen == KEY_LENGTH_128) { cipher = dlsym_EVP_aes_128_ctr(); } + } else if (alg == SM4_CTR) { + if (keyLen == KEY_LENGTH_128) { +#if OPENSSL_VERSION_NUMBER >= 0x10101001L + cipher = dlsym_EVP_sm4_ctr(); +#endif + } } return cipher; } JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_init (JNIEnv *env, jobject object, jlong ctx, jint mode, jint alg, jint padding, - jbyteArray key, jbyteArray iv) + jbyteArray key, jbyteArray iv, jlong engine) { int jKeyLen = (*env)->GetArrayLength(env, key); int jIvLen = (*env)->GetArrayLength(env, iv); @@ -275,9 +372,10 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_init THROW(env, "java/lang/InternalError", "Cannot get bytes array for iv."); return (jlong)0; } - + + ENGINE *e = LONG_TO_ENGINE(engine); int rc = dlsym_EVP_CipherInit_ex(context, getEvpCipher(alg, jKeyLen), \ - NULL, (unsigned char *)jKey, (unsigned char *)jIv, mode == ENCRYPT_MODE); + e, (unsigned char *)jKey, (unsigned char *)jIv, mode == ENCRYPT_MODE); (*env)->ReleaseByteArrayElements(env, key, jKey, 0); (*env)->ReleaseByteArrayElements(env, iv, jIv, 0); if (rc == 0) { @@ -406,12 +504,17 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_doFinal } JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_clean - (JNIEnv *env, jobject object, jlong ctx) + (JNIEnv *env, jobject object, jlong ctx, jlong engine) { EVP_CIPHER_CTX *context = CONTEXT(ctx); if (context) { dlsym_EVP_CIPHER_CTX_free(context); } + + ENGINE *e = LONG_TO_ENGINE(engine); + if (e) { + dlsym_ENGINE_free(e); + } } JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibraryName diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/org_apache_hadoop_crypto.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/org_apache_hadoop_crypto.h index 0afab021dac3c..3f377cfb29bd1 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/org_apache_hadoop_crypto.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/org_apache_hadoop_crypto.h @@ -46,6 +46,11 @@ */ #define JLONG(context) ((jlong)((ptrdiff_t)(context))) +/** + * A helper macro to convert long to ENGINE. + */ +#define LONG_TO_ENGINE(engine) ((ENGINE*)((ptrdiff_t)(engine))) + #define KEY_LENGTH_128 16 #define KEY_LENGTH_256 32 #define IV_LENGTH 16 @@ -53,8 +58,9 @@ #define ENCRYPT_MODE 1 #define DECRYPT_MODE 0 -/** Currently only support AES/CTR/NoPadding. */ +/** Currently only support AES/CTR/NoPadding & SM4/CTR/NoPadding. */ #define AES_CTR 0 +#define SM4_CTR 1 #define NOPADDING 0 #define PKCSPADDING 1 diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c index 2c8af1b9115d5..a54997e3350fb 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c @@ -30,7 +30,7 @@ static jfieldID Lz4Compressor_uncompressedDirectBuf; static jfieldID Lz4Compressor_uncompressedDirectBufLen; static jfieldID Lz4Compressor_compressedDirectBuf; -static jfieldID Lz4Compressor_directBufferSize; +static jfieldID Lz4Compressor_dstCapacity; JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_initIDs @@ -44,8 +44,8 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_init Lz4Compressor_compressedDirectBuf = (*env)->GetFieldID(env, clazz, "compressedDirectBuf", "Ljava/nio/Buffer;"); - Lz4Compressor_directBufferSize = (*env)->GetFieldID(env, clazz, - "directBufferSize", "I"); + Lz4Compressor_dstCapacity = (*env)->GetFieldID(env, clazz, + "dstCapacity", "I"); } JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_compressBytesDirect @@ -57,7 +57,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp jobject uncompressed_direct_buf = (*env)->GetObjectField(env, thisj, Lz4Compressor_uncompressedDirectBuf); jint uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, Lz4Compressor_uncompressedDirectBufLen); jobject compressed_direct_buf = (*env)->GetObjectField(env, thisj, Lz4Compressor_compressedDirectBuf); - jint compressed_direct_buf_len = (*env)->GetIntField(env, thisj, Lz4Compressor_directBufferSize); + jint compressed_direct_buf_len = (*env)->GetIntField(env, thisj, Lz4Compressor_dstCapacity); // Get the input direct buffer uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); @@ -73,7 +73,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp return (jint)0; } - compressed_direct_buf_len = LZ4_compress(uncompressed_bytes, compressed_bytes, uncompressed_direct_buf_len); + compressed_direct_buf_len = LZ4_compress_default(uncompressed_bytes, compressed_bytes, uncompressed_direct_buf_len, compressed_direct_buf_len); if (compressed_direct_buf_len < 0){ THROW(env, "java/lang/InternalError", "LZ4_compress failed"); } @@ -101,7 +101,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp jobject uncompressed_direct_buf = (*env)->GetObjectField(env, thisj, Lz4Compressor_uncompressedDirectBuf); jint uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, Lz4Compressor_uncompressedDirectBufLen); jobject compressed_direct_buf = (*env)->GetObjectField(env, thisj, Lz4Compressor_compressedDirectBuf); - jint compressed_direct_buf_len = (*env)->GetIntField(env, thisj, Lz4Compressor_directBufferSize); + jint compressed_direct_buf_len = (*env)->GetIntField(env, thisj, Lz4Compressor_dstCapacity); // Get the input direct buffer uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); @@ -117,7 +117,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp return (jint)0; } - compressed_direct_buf_len = LZ4_compressHC(uncompressed_bytes, compressed_bytes, uncompressed_direct_buf_len); + compressed_direct_buf_len = LZ4_compress_HC(uncompressed_bytes, compressed_bytes, uncompressed_direct_buf_len, compressed_direct_buf_len, 0); if (compressed_direct_buf_len < 0){ THROW(env, "java/lang/InternalError", "LZ4_compressHC failed"); } diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c index 5c51afe98ecea..9808d70aed032 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c @@ -1,6 +1,7 @@ /* LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-2014, Yann Collet. + Copyright (C) 2011-present, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -27,735 +28,1450 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 */ -/************************************** - Tuning parameters +/*-************************************ +* Tuning parameters **************************************/ /* - * HEAPMODE : + * LZ4_HEAPMODE : * Select how default compression functions will allocate memory for their hash table, - * in memory stack (0:default, fastest), or in memory heap (1:requires memory allocation (malloc)). + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#ifndef LZ4_HEAPMODE +# define LZ4_HEAPMODE 0 +#endif + +/* + * ACCELERATION_DEFAULT : + * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 */ -#define HEAPMODE 0 +#define ACCELERATION_DEFAULT 1 -/************************************** - CPU Feature Detection +/*-************************************ +* CPU Feature Detection **************************************/ -/* 32 or 64 bits ? */ -#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ - || defined(__64BIT__) || defined(__mips64) \ - || defined(__powerpc64__) || defined(__powerpc64le__) \ - || defined(__ppc64__) || defined(__ppc64le__) \ - || defined(__PPC64__) || defined(__PPC64LE__) \ - || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) \ - || defined(__s390x__) || defined(_LP64)) /* Detects 64 bits mode */ -# define LZ4_ARCH64 1 -#else -# define LZ4_ARCH64 0 +/* LZ4_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ +# if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define LZ4_FORCE_MEMORY_ACCESS 2 +# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__) +# define LZ4_FORCE_MEMORY_ACCESS 1 +# endif #endif -#define LZ4_32BITS (sizeof(void*)==4) -#define LZ4_64BITS (sizeof(void*)==8) /* - * Little Endian or Big Endian ? - * Overwrite the #define below if you know your architecture endianess + * LZ4_FORCE_SW_BITCOUNT + * Define this parameter if your target system or compiler does not support hardware bit count */ -#include /* Apparently required to detect endianess */ -#if defined (__GLIBC__) -# include -# if (__BYTE_ORDER == __BIG_ENDIAN) -# define LZ4_BIG_ENDIAN 1 -# endif -#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN)) -# define LZ4_BIG_ENDIAN 1 -#elif defined(__sparc) || defined(__sparc__) \ - || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \ - || defined(__hpux) || defined(__hppa) \ - || defined(_MIPSEB) || defined(__s390__) -# define LZ4_BIG_ENDIAN 1 -#else -/* Little Endian assumed. PDP Endian and other very rare endian format are unsupported. */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */ +# define LZ4_FORCE_SW_BITCOUNT #endif + + +/*-************************************ +* Dependency +**************************************/ /* - * Unaligned memory access is automatically enabled for "common" CPU, such as x86. - * For others CPU, such as ARM, the compiler may be more cautious, inserting unnecessary extra code to ensure aligned access property - * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance + * LZ4_SRC_INCLUDED: + * Amalgamation flag, whether lz4.c is included */ -#if defined(__ARM_FEATURE_UNALIGNED) -# define LZ4_FORCE_UNALIGNED_ACCESS 1 +#ifndef LZ4_SRC_INCLUDED +# define LZ4_SRC_INCLUDED 1 #endif -/* Define this parameter if your target system or compiler does not support hardware bit count */ -#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ -# define LZ4_FORCE_SW_BITCOUNT +#ifndef LZ4_STATIC_LINKING_ONLY +#define LZ4_STATIC_LINKING_ONLY #endif -/* - * BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE : - * This option may provide a small boost to performance for some big endian cpu, although probably modest. - * You may set this option to 1 if data will remain within closed environment. - * This option is useless on Little_Endian CPU (such as x86) - */ +#ifndef LZ4_DISABLE_DEPRECATE_WARNINGS +#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ +#endif -/* #define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1 */ +#define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */ +#include "lz4.h" +/* see also "memory routines" below */ -/************************************** - Compiler Options +/*-************************************ +* Compiler Options **************************************/ -#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -/* "restrict" is a known keyword */ -#else -# define restrict /* Disable restrict */ -#endif - #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include /* For Visual 2005 */ -# if LZ4_ARCH64 /* 64-bits */ -# pragma intrinsic(_BitScanForward64) /* For Visual 2005 */ -# pragma intrinsic(_BitScanReverse64) /* For Visual 2005 */ -# else /* 32-bits */ -# pragma intrinsic(_BitScanForward) /* For Visual 2005 */ -# pragma intrinsic(_BitScanReverse) /* For Visual 2005 */ -# endif +# include # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif +# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#endif /* _MSC_VER */ -#ifdef _MSC_VER /* Visual Studio */ -# define lz4_bswap16(x) _byteswap_ushort(x) +#ifndef LZ4_FORCE_INLINE +# ifdef _MSC_VER /* Visual Studio */ +# define LZ4_FORCE_INLINE static __forceinline +# else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define LZ4_FORCE_INLINE static inline +# endif +# else +# define LZ4_FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +# endif /* _MSC_VER */ +#endif /* LZ4_FORCE_INLINE */ + +/* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE + * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8, + * together with a simple 8-byte copy loop as a fall-back path. + * However, this optimization hurts the decompression speed by >30%, + * because the execution does not go to the optimized loop + * for typical compressible data, and all of the preamble checks + * before going to the fall-back path become useless overhead. + * This optimization happens only with the -O3 flag, and -O2 generates + * a simple 8-byte copy loop. + * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8 + * functions are annotated with __attribute__((optimize("O2"))), + * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute + * of LZ4_wildCopy8 does not affect the compression speed. + */ +#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__) +# define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2"))) +# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE #else -# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8))) +# define LZ4_FORCE_O2_GCC_PPC64LE +# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE static #endif -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) # define expect(expr,value) (__builtin_expect ((expr),(value)) ) #else # define expect(expr,value) (expr) #endif +#ifndef likely #define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely #define unlikely(expr) expect((expr) != 0, 0) +#endif -/************************************** - Memory routines +/*-************************************ +* Memory routines **************************************/ #include /* malloc, calloc, free */ -#define ALLOCATOR(n,s) calloc(n,s) -#define FREEMEM free +#define ALLOC(s) malloc(s) +#define ALLOC_AND_ZERO(s) calloc(1,s) +#define FREEMEM(p) free(p) #include /* memset, memcpy */ -#define MEM_INIT memset +#define MEM_INIT(p,v,s) memset((p),(v),(s)) -/************************************** - Includes +/*-************************************ +* Common Constants **************************************/ -#include "lz4.h" +#define MINMATCH 4 + +#define WILDCOPYLENGTH 8 +#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ +#define FASTLOOP_SAFE_DISTANCE 64 +static const int LZ4_minLength = (MFLIMIT+1); + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define LZ4_DISTANCE_ABSOLUTE_MAX 65535 +#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */ +# error "LZ4_DISTANCE_MAX is too big : must be <= 65535" +#endif +#define ML_BITS 4 +#define ML_MASK ((1U<= 199901L) /* C99 */ +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1) +# include +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) +# include +static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ ": "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + + +/*-************************************ +* Types +**************************************/ +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # include typedef uint8_t BYTE; typedef uint16_t U16; typedef uint32_t U32; typedef int32_t S32; typedef uint64_t U64; + typedef uintptr_t uptrval; #else typedef unsigned char BYTE; typedef unsigned short U16; typedef unsigned int U32; typedef signed int S32; typedef unsigned long long U64; + typedef size_t uptrval; /* generally true, except OpenVMS-64 */ #endif -#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) +#if defined(__x86_64__) + typedef U64 reg_t; /* 64-bits in x32 mode */ #else -# define _PACKED + typedef size_t reg_t; /* 32-bits in x32 mode */ #endif -#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif -#endif +typedef enum { + notLimited = 0, + limitedOutput = 1, + fillOutput = 2 +} limitedOutput_directive; -typedef struct { U16 v; } _PACKED U16_S; -typedef struct { U32 v; } _PACKED U32_S; -typedef struct { U64 v; } _PACKED U64_S; -typedef struct {size_t v;} _PACKED size_t_S; -#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# if defined(__SUNPRO_C) || defined(__SUNPRO_CC) -# pragma pack(0) -# else -# pragma pack(pop) -# endif -#endif +/*-************************************ +* Reading and writing into memory +**************************************/ +static unsigned LZ4_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} -#define A16(x) (((U16_S *)(x))->v) -#define A32(x) (((U32_S *)(x))->v) -#define A64(x) (((U64_S *)(x))->v) -#define AARCH(x) (((size_t_S *)(x))->v) +#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) +/* lie to the compiler about data alignment; use with caution */ -/************************************** - Constants -**************************************/ -#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) -#define HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) -#define HASH_SIZE_U32 (1 << LZ4_HASHLOG) +static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } +static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } +static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } -#define MINMATCH 4 +static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } -#define COPYLENGTH 8 -#define LASTLITERALS 5 -#define MFLIMIT (COPYLENGTH+MINMATCH) -static const int LZ4_minLength = (MFLIMIT+1); +#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign; -#define LZ4_64KLIMIT ((64 KB) + (MFLIMIT-1)) -#define SKIPSTRENGTH 6 /* Increasing this value will make the compression run slower on incompressible data */ +static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } +static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; } -#define MAXD_LOG 16 -#define MAX_DISTANCE ((1 << MAXD_LOG) - 1) +static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } +static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } -#define ML_BITS 4 -#define ML_MASK ((1U<=e; */ -#else -# define LZ4_WILDCOPY(d,s,e) { if (likely(e-d <= 8)) LZ4_COPY8(d,s) else do { LZ4_COPY8(d,s) } while (d>8); + } +} + +/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ +LZ4_FORCE_O2_INLINE_GCC_PPC64LE +void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { memcpy(d,s,8); d+=8; s+=8; } while (d>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll(val) >> 3); -# else - int r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif -# else -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward64( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll(val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif -# endif + LZ4_wildCopy8(dstPtr, srcPtr, dstEnd); } -#else +/* customized variant of memcpy, which can overwrite up to 32 bytes beyond dstEnd + * this version copies two times 16 bytes (instead of one time 32 bytes) + * because it must be compatible with offsets >= 16. */ +LZ4_FORCE_O2_INLINE_GCC_PPC64LE void +LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; -static int LZ4_NbCommonBytes (register U32 val) -{ -# if defined(LZ4_BIG_ENDIAN) -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz(val) >> 3); -# else - int r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif -# else -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz(val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif -# endif + do { memcpy(d,s,16); memcpy(d+16,s+16,16); d+=32; s+=32; } while (d= dstPtr + MINMATCH + * - there is at least 8 bytes available to write after dstEnd */ +LZ4_FORCE_O2_INLINE_GCC_PPC64LE void +LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) +{ + BYTE v[8]; + + assert(dstEnd >= dstPtr + MINMATCH); + LZ4_write32(dstPtr, 0); /* silence an msan warning when offset==0 */ + + switch(offset) { + case 1: + memset(v, *srcPtr, 8); + break; + case 2: + memcpy(v, srcPtr, 2); + memcpy(&v[2], srcPtr, 2); + memcpy(&v[4], &v[0], 4); + break; + case 4: + memcpy(v, srcPtr, 4); + memcpy(&v[4], srcPtr, 4); + break; + default: + LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); + return; + } + + memcpy(dstPtr, v, 8); + dstPtr += 8; + while (dstPtr < dstEnd) { + memcpy(dstPtr, v, 8); + dstPtr += 8; + } +} #endif -/******************************** - Compression functions -********************************/ +/*-************************************ +* Common functions +**************************************/ +static unsigned LZ4_NbCommonBytes (reg_t val) +{ + if (LZ4_isLittleEndian()) { + if (sizeof(val)==8) { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64( &r, (U64)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctzll((U64)val) >> 3; +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, + 0, 3, 1, 3, 1, 4, 2, 7, + 0, 2, 3, 6, 1, 5, 3, 5, + 1, 3, 4, 4, 2, 5, 6, 7, + 7, 0, 1, 2, 3, 3, 4, 6, + 2, 6, 5, 5, 3, 4, 5, 6, + 7, 1, 2, 4, 6, 4, 4, 5, + 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, (U32)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctz((U32)val) >> 3; +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, + 3, 2, 2, 1, 3, 2, 0, 1, + 3, 3, 1, 2, 2, 2, 2, 0, + 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else /* Big Endian CPU */ { + if (sizeof(val)==8) { /* 64-bits */ +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clzll((U64)val) >> 3; +# else + static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. + Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. + Note that this code path is never triggered in 32-bits mode. */ + unsigned r; + if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clz((U32)val) >> 3; +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } + } +} + +#define STEPSIZE sizeof(reg_t) +LZ4_FORCE_INLINE +unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + if (likely(pIn < pInLimit-(STEPSIZE-1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { + pIn+=STEPSIZE; pMatch+=STEPSIZE; + } else { + return LZ4_NbCommonBytes(diff); + } } + + while (likely(pIn < pInLimit-(STEPSIZE-1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; } + pIn += LZ4_NbCommonBytes(diff); + return (unsigned)(pIn - pStart); + } + + if ((STEPSIZE==8) && (pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn compression run slower on incompressible data */ + + +/*-************************************ +* Local Structures and types +**************************************/ +typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; + +/** + * This enum distinguishes several different modes of accessing previous + * content in the stream. + * + * - noDict : There is no preceding content. + * - withPrefix64k : Table entries up to ctx->dictSize before the current blob + * blob being compressed are valid and refer to the preceding + * content (of length ctx->dictSize), which is available + * contiguously preceding in memory the content currently + * being compressed. + * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere + * else in memory, starting at ctx->dictionary with length + * ctx->dictSize. + * - usingDictCtx : Like usingExtDict, but everything concerning the preceding + * content is in a separate context, pointed to by + * ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table + * entries in the current context that refer to positions + * preceding the beginning of the current compression are + * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx + * ->dictSize describe the location and size of the preceding + * content, and matches are found by looking in the ctx + * ->dictCtx->hashTable. + */ +typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive; +typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; + + +/*-************************************ +* Local Utils +**************************************/ int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } +int LZ4_sizeofState() { return LZ4_STREAMSIZE; } -static int LZ4_hashSequence(U32 sequence, tableType_t tableType) + +/*-************************************ +* Internal Definitions used in Tests +**************************************/ +#if defined (__cplusplus) +extern "C" { +#endif + +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize); + +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize); + +#if defined (__cplusplus) +} +#endif + +/*-****************************** +* Compression functions +********************************/ +static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) { if (tableType == byU16) - return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); else - return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); + return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); +} + +static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) +{ + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + if (LZ4_isLittleEndian()) { + const U64 prime5bytes = 889523592379ULL; + return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); + } else { + const U64 prime8bytes = 11400714785074694791ULL; + return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); + } +} + +LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) +{ + if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); + return LZ4_hash4(LZ4_read32(p), tableType); } -static int LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); } +static void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: { /* illegal! */ assert(0); return; } + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; } + } +} -static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) { switch (tableType) { - case byPtr: { const BYTE** hashTable = (const BYTE**) tableBase; hashTable[h] = p; break; } - case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); break; } - case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); break; } + default: /* fallthrough */ + case clearedTable: /* fallthrough */ + case byPtr: { /* illegal! */ assert(0); return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; } + case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; } } } -static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +static void LZ4_putPositionOnHash(const BYTE* p, U32 h, + void* tableBase, tableType_t const tableType, + const BYTE* srcBase) { - U32 h = LZ4_hashPosition(p, tableType); + switch (tableType) + { + case clearedTable: { /* illegal! */ assert(0); return; } + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } + } +} + +LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 const h = LZ4_hashPosition(p, tableType); LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); } -static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +/* LZ4_getIndexOnHash() : + * Index of match position registered in hash table. + * hash position must be calculated by using base+index, or dictBase+index. + * Assumption 1 : only valid if tableType == byU32 or byU16. + * Assumption 2 : h is presumed valid (within limits of hash table) + */ +static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) { - if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } - if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } - { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ + LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); + if (tableType == byU32) { + const U32* const hashTable = (const U32*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-2))); + return hashTable[h]; + } + if (tableType == byU16) { + const U16* const hashTable = (const U16*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-1))); + return hashTable[h]; + } + assert(0); return 0; /* forbidden case */ } -static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase) { - U32 h = LZ4_hashPosition(p, tableType); - return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); + if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; } + if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; return hashTable[h] + srcBase; } + { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } -static unsigned LZ4_count(const BYTE* pIn, const BYTE* pRef, const BYTE* pInLimit) +LZ4_FORCE_INLINE const BYTE* +LZ4_getPosition(const BYTE* p, + const void* tableBase, tableType_t tableType, + const BYTE* srcBase) { - const BYTE* const pStart = pIn; + U32 const h = LZ4_hashPosition(p, tableType); + return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); +} - while (likely(pIndirty) { + DEBUGLOG(5, "LZ4_prepareTable: Full reset for %p", cctx); + MEM_INIT(cctx, 0, sizeof(LZ4_stream_t_internal)); + return; } - if (LZ4_64BITS) if ((pIn<(pInLimit-3)) && (A32(pRef) == A32(pIn))) { pIn+=4; pRef+=4; } - if ((pIn<(pInLimit-1)) && (A16(pRef) == A16(pIn))) { pIn+=2; pRef+=2; } - if ((pIntableType != clearedTable) { + assert(inputSize >= 0); + if (cctx->tableType != tableType + || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) + || ((tableType == byU32) && cctx->currentOffset > 1 GB) + || tableType == byPtr + || inputSize >= 4 KB) + { + DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx); + MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); + cctx->currentOffset = 0; + cctx->tableType = clearedTable; + } else { + DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)"); + } + } + /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, is faster + * than compressing without a gap. However, compressing with + * currentOffset == 0 is faster still, so we preserve that case. + */ + if (cctx->currentOffset != 0 && tableType == byU32) { + DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); + cctx->currentOffset += 64 KB; + } -static int LZ4_compress_generic( - void* ctx, - const char* source, - char* dest, - int inputSize, - int maxOutputSize, + /* Finally, clear history */ + cctx->dictCtx = NULL; + cctx->dictionary = NULL; + cctx->dictSize = 0; +} - limitedOutput_directive outputLimited, - tableType_t tableType, - dict_directive dict, - dictIssue_directive dictIssue) +/** LZ4_compress_generic() : + inlined, to ensure branches are decided at compilation time */ +LZ4_FORCE_INLINE int LZ4_compress_generic( + LZ4_stream_t_internal* const cctx, + const char* const source, + char* const dest, + const int inputSize, + int *inputConsumed, /* only written when outputDirective == fillOutput */ + const int maxOutputSize, + const limitedOutput_directive outputDirective, + const tableType_t tableType, + const dict_directive dictDirective, + const dictIssue_directive dictIssue, + const int acceleration) { - LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx; - + int result; const BYTE* ip = (const BYTE*) source; - const BYTE* base; + + U32 const startIndex = cctx->currentOffset; + const BYTE* base = (const BYTE*) source - startIndex; const BYTE* lowLimit; - const BYTE* const lowRefLimit = ip - dictPtr->dictSize; - const BYTE* const dictionary = dictPtr->dictionary; - const BYTE* const dictEnd = dictionary + dictPtr->dictSize; - const size_t dictDelta = dictEnd - (const BYTE*)source; + + const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx; + const BYTE* const dictionary = + dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary; + const U32 dictSize = + dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize; + const U32 dictDelta = (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with index in current context */ + + int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); + U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ + const BYTE* const dictEnd = dictionary + dictSize; const BYTE* anchor = (const BYTE*) source; const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1; const BYTE* const matchlimit = iend - LASTLITERALS; + /* the dictCtx currentOffset is indexed on the start of the dictionary, + * while a dictionary in the current context precedes the currentOffset */ + const BYTE* dictBase = (dictDirective == usingDictCtx) ? + dictionary + dictSize - dictCtx->currentOffset : + dictionary + dictSize - startIndex; + BYTE* op = (BYTE*) dest; BYTE* const olimit = op + maxOutputSize; - const int skipStrength = SKIPSTRENGTH; + U32 offset = 0; U32 forwardH; - size_t refDelta=0; - /* Init conditions */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ - switch(dict) - { - case noDict: - default: - base = (const BYTE*)source; - lowLimit = (const BYTE*)source; - break; - case withPrefix64k: - base = (const BYTE*)source - dictPtr->currentOffset; - lowLimit = (const BYTE*)source - dictPtr->dictSize; - break; - case usingExtDict: - base = (const BYTE*)source - dictPtr->currentOffset; - lowLimit = (const BYTE*)source; - break; + DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType); + /* If init conditions are not met, we don't have to mark stream + * as having dirty context, since no action was taken yet */ + if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; } /* Impossible to store anything */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported inputSize, too large (or negative) */ + if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) { return 0; } /* Size too large (not within 64K limit) */ + if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ + assert(acceleration >= 1); + + lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); + + /* Update context state */ + if (dictDirective == usingDictCtx) { + /* Subsequent linked blocks can't use the dictionary. */ + /* Instead, they use the block we just compressed. */ + cctx->dictCtx = NULL; + cctx->dictSize = (U32)inputSize; + } else { + cctx->dictSize += (U32)inputSize; } - if ((tableType == byU16) && (inputSize>=(int)LZ4_64KLIMIT)) return 0; /* Size too large (not within 64K limit) */ - if (inputSizecurrentOffset += (U32)inputSize; + cctx->tableType = (U16)tableType; + + if (inputSizehashTable, tableType, base); ip++; forwardH = LZ4_hashPosition(ip, tableType); /* Main Loop */ - for ( ; ; ) - { - const BYTE* ref; + for ( ; ; ) { + const BYTE* match; BYTE* token; - { - const BYTE* forwardIp = ip; - unsigned step=1; - unsigned searchMatchNb = (1U << skipStrength); + const BYTE* filledIp; - /* Find a match */ + /* Find a match */ + if (tableType == byPtr) { + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; do { - U32 h = forwardH; + U32 const h = forwardH; ip = forwardIp; forwardIp += step; - step = searchMatchNb++ >> skipStrength; + step = (searchMatchNb++ >> LZ4_skipTrigger); - if (unlikely(forwardIp > mflimit)) goto _last_literals; + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); - ref = LZ4_getPositionOnHash(h, ctx, tableType, base); - if (dict==usingExtDict) - { - if (ref<(const BYTE*)source) - { - refDelta = dictDelta; + match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + + } while ( (match+LZ4_DISTANCE_MAX < ip) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + + } else { /* byU32, byU16 */ + + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + U32 const current = (U32)(forwardIp - base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex <= current); + assert(forwardIp - base < (ptrdiff_t)(2 GB - 1)); + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + matchIndex += dictDelta; /* make dictCtx index comparable with current context */ lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; } - else - { - refDelta = 0; + } else if (dictDirective==usingExtDict) { + if (matchIndex < startIndex) { + DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); + assert(startIndex - matchIndex >= MINMATCH); + match = dictBase + matchIndex; + lowLimit = dictionary; + } else { + match = base + matchIndex; lowLimit = (const BYTE*)source; } + } else { /* single continuous memory segment */ + match = base + matchIndex; } forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx, tableType, base); + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + + DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex); + if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */ + assert(matchIndex < current); + if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX)) + && (matchIndex+LZ4_DISTANCE_MAX < current)) { + continue; + } /* too far */ + assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */ + + if (LZ4_read32(match) == LZ4_read32(ip)) { + if (maybe_extMem) offset = current - matchIndex; + break; /* match found */ + } - } while ( ((dictIssue==dictSmall) ? (ref < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (ref + MAX_DISTANCE < ip)) - || (A32(ref+refDelta) != A32(ip)) ); + } while(1); } /* Catch up */ - while ((ip>anchor) && (ref+refDelta > lowLimit) && (unlikely(ip[-1]==ref[refDelta-1]))) { ip--; ref--; } + filledIp = ip; + while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } - { - /* Encode Literal length */ - unsigned litLength = (unsigned)(ip - anchor); + /* Encode Literals */ + { unsigned const litLength = (unsigned)(ip - anchor); token = op++; - if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) - return 0; /* Check output limit */ - if (litLength>=RUN_MASK) - { - int len = (int)litLength-RUN_MASK; - *token=(RUN_MASK< olimit)) ) { + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + if ((outputDirective == fillOutput) && + (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { + op--; + goto _last_literals; + } + if (litLength >= RUN_MASK) { + int len = (int)(litLength - RUN_MASK); + *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; *op++ = (BYTE)len; } else *token = (BYTE)(litLength< olimit)) { + /* the match was too close to the end, rewind and go to last literals */ + op = token; + goto _last_literals; + } + /* Encode Offset */ - LZ4_WRITE_LITTLEENDIAN_16(op, (U16)(ip-ref)); + if (maybe_extMem) { /* static test */ + DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source)); + assert(offset <= LZ4_DISTANCE_MAX && offset > 0); + LZ4_writeLE16(op, (U16)offset); op+=2; + } else { + DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match)); + assert(ip-match <= LZ4_DISTANCE_MAX); + LZ4_writeLE16(op, (U16)(ip - match)); op+=2; + } /* Encode MatchLength */ - { - unsigned matchLength; + { unsigned matchCode; - if ((dict==usingExtDict) && (lowLimit==dictionary)) - { - const BYTE* limit; - ref += refDelta; - limit = ip + (dictEnd-ref); + if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) + && (lowLimit==dictionary) /* match within extDict */ ) { + const BYTE* limit = ip + (dictEnd-match); + assert(dictEnd > match); if (limit > matchlimit) limit = matchlimit; - matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, limit); - ip += MINMATCH + matchLength; - if (ip==limit) - { - unsigned more = LZ4_count(ip, (const BYTE*)source, matchlimit); - matchLength += more; + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); + ip += (size_t)matchCode + MINMATCH; + if (ip==limit) { + unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit); + matchCode += more; ip += more; } - } - else - { - matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, matchlimit); - ip += MINMATCH + matchLength; + DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH); + } else { + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += (size_t)matchCode + MINMATCH; + DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH); } - if (matchLength>=ML_MASK) - { - if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > olimit))) - return 0; /* Check output limit */ - *token += ML_MASK; - matchLength -= ML_MASK; - for (; matchLength >= 510 ; matchLength-=510) { *op++ = 255; *op++ = 255; } - if (matchLength >= 255) { matchLength-=255; *op++ = 255; } - *op++ = (BYTE)matchLength; + if ((outputDirective) && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) { + if (outputDirective == fillOutput) { + /* Match description too long : reduce it */ + U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255; + ip -= matchCode - newMatchCode; + assert(newMatchCode < matchCode); + matchCode = newMatchCode; + if (unlikely(ip <= filledIp)) { + /* We have already filled up to filledIp so if ip ends up less than filledIp + * we have positions in the hash table beyond the current position. This is + * a problem if we reuse the hash table. So we have to remove these positions + * from the hash table. + */ + const BYTE* ptr; + DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip)); + for (ptr = ip; ptr <= filledIp; ++ptr) { + U32 const h = LZ4_hashPosition(ptr, tableType); + LZ4_clearHash(h, cctx->hashTable, tableType); + } + } + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } } - else *token += (BYTE)(matchLength); + if (matchCode >= ML_MASK) { + *token += ML_MASK; + matchCode -= ML_MASK; + LZ4_write32(op, 0xFFFFFFFF); + while (matchCode >= 4*255) { + op+=4; + LZ4_write32(op, 0xFFFFFFFF); + matchCode -= 4*255; + } + op += matchCode / 255; + *op++ = (BYTE)(matchCode % 255); + } else + *token += (BYTE)(matchCode); } + /* Ensure we have enough space for the last literals. */ + assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit)); anchor = ip; /* Test end of chunk */ - if (ip > mflimit) break; + if (ip >= mflimitPlusOne) break; /* Fill table */ - LZ4_putPosition(ip-2, ctx, tableType, base); + LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); /* Test next position */ - ref = LZ4_getPosition(ip, ctx, tableType, base); - if (dict==usingExtDict) - { - if (ref<(const BYTE*)source) - { - refDelta = dictDelta; - lowLimit = dictionary; + if (tableType == byPtr) { + + match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + if ( (match+LZ4_DISTANCE_MAX >= ip) + && (LZ4_read32(match) == LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + } else { /* byU32, byU16 */ + + U32 const h = LZ4_hashPosition(ip, tableType); + U32 const current = (U32)(ip-base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex < current); + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + matchIndex += dictDelta; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else if (dictDirective==usingExtDict) { + if (matchIndex < startIndex) { + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else { /* single memory segment */ + match = base + matchIndex; } - else - { - refDelta = 0; - lowLimit = (const BYTE*)source; + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + assert(matchIndex < current); + if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) + && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) + && (LZ4_read32(match) == LZ4_read32(ip)) ) { + token=op++; + *token=0; + if (maybe_extMem) offset = current - matchIndex; + DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", + (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source)); + goto _next_match; } } - LZ4_putPosition(ip, ctx, tableType, base); - if ( ((dictIssue==dictSmall) ? (ref>=lowRefLimit) : 1) - && (ref+MAX_DISTANCE>=ip) - && (A32(ref+refDelta)==A32(ip)) ) - { token=op++; *token=0; goto _next_match; } /* Prepare next loop */ forwardH = LZ4_hashPosition(++ip, tableType); + } _last_literals: /* Encode Last Literals */ - { - int lastRun = (int)(iend - anchor); - if ((outputLimited) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) - return 0; /* Check output limit */ - if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } - else *op++ = (BYTE)(lastRun< olimit)) { + if (outputDirective == fillOutput) { + /* adapt lastRun to fill 'dst' */ + assert(olimit >= op); + lastRun = (size_t)(olimit-op) - 1; + lastRun -= (lastRun+240)/255; + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } + if (lastRun >= RUN_MASK) { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRun< 0); + return result; +} + + +int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse; + assert(ctx != NULL); + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + if (maxOutputSize >= LZ4_compressBound(inputSize)) { + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } +} + +/** + * LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of + * "correctly initialized"). + */ +int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) +{ + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + + if (dstCapacity >= LZ4_compressBound(srcSize)) { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } } -int LZ4_compress(const char* source, char* dest, int inputSize) +int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { -#if (HEAPMODE) - void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U32, 4); /* Aligned on 4-bytes boundaries */ + int result; +#if (LZ4_HEAPMODE) + LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctxPtr == NULL) return 0; #else - U32 ctx[LZ4_STREAMSIZE_U32] = {0}; /* Ensure data is aligned on 4-bytes boundaries */ + LZ4_stream_t ctx; + LZ4_stream_t* const ctxPtr = &ctx; #endif - int result; - - if (inputSize < (int)LZ4_64KLIMIT) - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue); - else - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, LZ4_64BITS ? byU32 : byPtr, noDict, noDictIssue); + result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); -#if (HEAPMODE) - FREEMEM(ctx); +#if (LZ4_HEAPMODE) + FREEMEM(ctxPtr); #endif return result; } -int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) + +int LZ4_compress_default(const char* src, char* dst, int srcSize, int maxOutputSize) { -#if (HEAPMODE) - void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U32, 4); /* Aligned on 4-bytes boundaries */ + return LZ4_compress_fast(src, dst, srcSize, maxOutputSize, 1); +} + + +/* hidden debug function */ +/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ +int LZ4_compress_fast_force(const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) +{ + LZ4_stream_t ctx; + LZ4_initStream(&ctx, sizeof(ctx)); + + if (srcSize < LZ4_64Klimit) { + return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, byU16, noDict, noDictIssue, acceleration); + } else { + tableType_t const addrMode = (sizeof(void*) > 4) ? byU32 : byPtr; + return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, addrMode, noDict, noDictIssue, acceleration); + } +} + + +/* Note!: This function leaves the stream in an unclean/broken state! + * It is not safe to subsequently use the same state with a _fastReset() or + * _continue() call without resetting it. */ +static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ + void* const s = LZ4_initStream(state, sizeof (*state)); + assert(s != NULL); (void)s; + + if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ + return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); + } else { + if (*srcSizePtr < LZ4_64Klimit) { + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1); + } else { + tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, 1); + } } +} + + +int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ +#if (LZ4_HEAPMODE) + LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctx == NULL) return 0; #else - U32 ctx[LZ4_STREAMSIZE_U32] = {0}; /* Ensure data is aligned on 4-bytes boundaries */ + LZ4_stream_t ctxBody; + LZ4_stream_t* ctx = &ctxBody; #endif - int result; - if (inputSize < (int)LZ4_64KLIMIT) - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue); - else - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64BITS ? byU32 : byPtr, noDict, noDictIssue); + int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); -#if (HEAPMODE) +#if (LZ4_HEAPMODE) FREEMEM(ctx); #endif return result; } -/***************************************** - Experimental : Streaming functions -*****************************************/ -/* - * LZ4_initStream - * Use this function once, to init a newly allocated LZ4_stream_t structure - * Return : 1 if OK, 0 if error - */ +/*-****************************** +* Streaming functions +********************************/ + +LZ4_stream_t* LZ4_createStream(void) +{ + LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); + LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ + DEBUGLOG(4, "LZ4_createStream %p", lz4s); + if (lz4s == NULL) return NULL; + LZ4_initStream(lz4s, sizeof(*lz4s)); + return lz4s; +} + +#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : + it reports an aligment of 8-bytes, + while actually aligning LZ4_stream_t on 4 bytes. */ +static size_t LZ4_stream_t_alignment(void) +{ + struct { char c; LZ4_stream_t t; } t_a; + return sizeof(t_a) - sizeof(t_a.t); +} +#endif + +LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) +{ + DEBUGLOG(5, "LZ4_initStream"); + if (buffer == NULL) { return NULL; } + if (size < sizeof(LZ4_stream_t)) { return NULL; } +#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : + it reports an aligment of 8-bytes, + while actually aligning LZ4_stream_t on 4 bytes. */ + if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) { return NULL; } /* alignment check */ +#endif + MEM_INIT(buffer, 0, sizeof(LZ4_stream_t)); + return (LZ4_stream_t*)buffer; +} + +/* resetStream is now deprecated, + * prefer initStream() which is more general */ void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { + DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); } -LZ4_stream_t* LZ4_createStream(void) -{ - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(4, LZ4_STREAMSIZE_U32); - LZ4_resetStream(lz4s); - return lz4s; +void LZ4_resetStream_fast(LZ4_stream_t* ctx) { + LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); } int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { + if (!LZ4_stream) return 0; /* support free on NULL */ + DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream); FREEMEM(LZ4_stream); return (0); } +#define HASH_UNIT sizeof(reg_t) int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) { - LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; + LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; + const tableType_t tableType = byU32; const BYTE* p = (const BYTE*)dictionary; const BYTE* const dictEnd = p + dictSize; const BYTE* base; - LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ - if (dict->initCheck) LZ4_resetStream(LZ4_dict); /* Uninitialized structure detected */ + DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict); - if (dictSize < MINMATCH) - { - dict->dictionary = NULL; - dict->dictSize = 0; - return 1; + /* It's necessary to reset the context, + * and not just continue it with prepareTable() + * to avoid any risk of generating overflowing matchIndex + * when compressing using this dictionary */ + LZ4_resetStream(LZ4_dict); + + /* We always increment the offset by 64 KB, since, if the dict is longer, + * we truncate it to the last 64k, and if it's shorter, we still want to + * advance by a whole window length so we can provide the guarantee that + * there are only valid offsets in the window, which allows an optimization + * in LZ4_compress_fast_continue() where it uses noDictIssue even when the + * dictionary isn't a full 64k. */ + dict->currentOffset += 64 KB; + + if (dictSize < (int)HASH_UNIT) { + return 0; } - if (p <= dictEnd - 64 KB) p = dictEnd - 64 KB; - base = p - dict->currentOffset; + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + base = dictEnd - dict->currentOffset; dict->dictionary = p; dict->dictSize = (U32)(dictEnd - p); - dict->currentOffset += dict->dictSize; + dict->tableType = tableType; - while (p <= dictEnd-MINMATCH) - { - LZ4_putPosition(p, dict, byU32, base); + while (p <= dictEnd-HASH_UNIT) { + LZ4_putPosition(p, dict->hashTable, tableType, base); p+=3; } - return 1; + return (int)dict->dictSize; +} + +void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) { + const LZ4_stream_t_internal* dictCtx = dictionaryStream == NULL ? NULL : + &(dictionaryStream->internal_donotuse); + + DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", + workingStream, dictionaryStream, + dictCtx != NULL ? dictCtx->dictSize : 0); + + /* Calling LZ4_resetStream_fast() here makes sure that changes will not be + * erased by subsequent calls to LZ4_resetStream_fast() in case stream was + * marked as having dirty context, e.g. requiring full reset. + */ + LZ4_resetStream_fast(workingStream); + + if (dictCtx != NULL) { + /* If the current offset is zero, we will never look in the + * external dictionary context, since there is no value a table + * entry can take that indicate a miss. In that case, we need + * to bump the offset to something non-zero. + */ + if (workingStream->internal_donotuse.currentOffset == 0) { + workingStream->internal_donotuse.currentOffset = 64 KB; + } + + /* Don't actually attach an empty dictionary. + */ + if (dictCtx->dictSize == 0) { + dictCtx = NULL; + } + } + workingStream->internal_donotuse.dictCtx = dictCtx; } -static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) { - if ((LZ4_dict->currentOffset > 0x80000000) || - ((size_t)LZ4_dict->currentOffset > (size_t)src)) /* address space overflow */ - { + assert(nextSize >= 0); + if (LZ4_dict->currentOffset + (unsigned)nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ /* rescale hash table */ - U32 delta = LZ4_dict->currentOffset - 64 KB; + U32 const delta = LZ4_dict->currentOffset - 64 KB; const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; int i; - for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; else LZ4_dict->hashTable[i] -= delta; } @@ -766,22 +1482,33 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) } -FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* source, char* dest, int inputSize, - int maxOutputSize, limitedOutput_directive limit) +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, + const char* source, char* dest, + int inputSize, int maxOutputSize, + int acceleration) { - LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + const tableType_t tableType = byU32; + LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; + const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); - const BYTE* smallest = (const BYTE*) source; - if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; - LZ4_renormDictT(streamPtr, smallest); + if (streamPtr->dirty) { return 0; } /* Uninitialized structure detected */ + LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + + /* invalidate tiny dictionaries */ + if ( (streamPtr->dictSize-1 < 4-1) /* intentional underflow */ + && (dictEnd != (const BYTE*)source) ) { + DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary); + streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE*)source; + dictEnd = (const BYTE*)source; + } /* Check overlapping input/dictionary space */ - { - const BYTE* sourceEnd = (const BYTE*) source + inputSize; - if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) - { + { const BYTE* sourceEnd = (const BYTE*) source + inputSize; + if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { streamPtr->dictSize = (U32)(dictEnd - sourceEnd); if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; @@ -790,71 +1517,81 @@ FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* so } /* prefix mode : source data follows dictionary */ - if (dictEnd == (const BYTE*)source) - { - int result; + if (dictEnd == (const BYTE*)source) { if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, dictSmall); + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, noDictIssue); - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration); } /* external dictionary mode */ - { - int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, dictSmall); - else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, noDictIssue); + { int result; + if (streamPtr->dictCtx) { + /* We depend here on the fact that dictCtx'es (produced by + * LZ4_loadDict) guarantee that their tables contain no references + * to offsets between dictCtx->currentOffset - 64 KB and + * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe + * to use noDictIssue even when the dict isn't a full 64 KB. + */ + if (inputSize > 4 KB) { + /* For compressing large blobs, it is faster to pay the setup + * cost to copy the dictionary's tables into the active context, + * so that the compression loop is only looking into one table. + */ + memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t)); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); + } + } else { + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } + } streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; return result; } } -int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) -{ - return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, 0, notLimited); -} - -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput); -} - -/* Hidden debug function, to force separate dictionary mode */ -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) +/* Hidden debug function, to force-test external dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize) { - LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict; + LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; int result; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - const BYTE* smallest = dictEnd; - if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest); + LZ4_renormDictT(streamPtr, srcSize); - result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue); + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + } streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; + streamPtr->dictSize = (U32)srcSize; return result; } +/*! LZ4_saveDict() : + * If previously compressed data block is not guaranteed to remain available at its memory location, + * save it into a safer place (char* safeBuffer). + * Note : you don't need to call LZ4_loadDict() afterwards, + * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). + * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + */ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) { - LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; - const BYTE* previousDictEnd = dict->dictionary + dict->dictSize; + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; - if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; + if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } memmove(safeBuffer, previousDictEnd - dictSize, dictSize); @@ -866,281 +1603,694 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) -/**************************** - Decompression functions -****************************/ -/* - * This generic decompression function cover all use cases. - * It shall be instantiated several times, using different sets of directives - * Note that it is essential this generic function is really inlined, - * in order to remove useless branches during compilation optimization. +/*-******************************* + * Decompression functions + ********************************/ + +typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; +typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; + +#undef MIN +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +/* Read the variable-length literal or match length. + * + * ip - pointer to use as input. + * lencheck - end ip. Return an error if ip advances >= lencheck. + * loop_check - check ip >= lencheck in body of loop. Returns loop_error if so. + * initial_check - check ip >= lencheck before start of loop. Returns initial_error if so. + * error (output) - error code. Should be set to 0 before call. + */ +typedef enum { loop_error = -2, initial_error = -1, ok = 0 } variable_length_error; +LZ4_FORCE_INLINE unsigned +read_variable_length(const BYTE**ip, const BYTE* lencheck, int loop_check, int initial_check, variable_length_error* error) +{ + unsigned length = 0; + unsigned s; + if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ + *error = initial_error; + return length; + } + do { + s = **ip; + (*ip)++; + length += s; + if (loop_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ + *error = loop_error; + return length; + } + } while (s==255); + + return length; +} + +/*! LZ4_decompress_generic() : + * This generic decompression function covers all use cases. + * It shall be instantiated several times, using different sets of directives. + * Note that it is important for performance that this function really get inlined, + * in order to remove useless branches during compilation optimization. */ -FORCE_INLINE int LZ4_decompress_generic( - const char* source, - char* dest, - int inputSize, - int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ - - int endOnInput, /* endOnOutputSize, endOnInputSize */ - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const char* dictStart, /* only if dict==usingExtDict */ - int dictSize /* note : = 0 if noDict */ +LZ4_FORCE_INLINE int +LZ4_decompress_generic( + const char* const src, + char* const dst, + int srcSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ + + endCondition_directive endOnInput, /* endOnOutputSize, endOnInputSize */ + earlyEnd_directive partialDecoding, /* full, partial */ + dict_directive dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ ) { - /* Local Variables */ - const BYTE* restrict ip = (const BYTE*) source; - const BYTE* ref; - const BYTE* const iend = ip + inputSize; + if (src == NULL) { return -1; } - BYTE* op = (BYTE*) dest; - BYTE* const oend = op + outputSize; - BYTE* cpy; - BYTE* oexit = op + targetOutputSize; - const BYTE* const lowLimit = (const BYTE*)dest - dictSize; + { const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; - const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const size_t dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; - const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3}; + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + outputSize; + BYTE* cpy; - const int safeDecode = (endOnInput==endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; + const int safeDecode = (endOnInput==endOnInputSize); + const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); - /* Special cases */ - if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ - if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); + /* Set up the "end" pointers for the shortcut. */ + const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; + const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; - /* Main Loop */ - while (1) - { + const BYTE* match; + size_t offset; unsigned token; size_t length; - /* get runlength */ - token = *ip++; - if ((length=(token>>ML_BITS)) == RUN_MASK) - { - unsigned s; - do - { - s = *ip++; - length += s; - } - while (likely((endOnInput)?ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-COPYLENGTH))) - { - if (partialDecoding) - { - if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + /* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */ + while (1) { + /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ + assert(oend - op >= FASTLOOP_SAFE_DISTANCE); + if (endOnInput) { assert(ip < iend); } + token = *ip++; + length = token >> ML_BITS; /* literal length */ + + assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ + + /* decode literal length */ + if (length == RUN_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); + if (error == initial_error) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + + /* copy literals */ + cpy = op+length; + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if (endOnInput) { /* LZ4_decompress_safe() */ + if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } + LZ4_wildCopy32(op, ip, cpy); + } else { /* LZ4_decompress_fast() */ + if (cpy>oend-8) { goto safe_literal_copy; } + LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : + * it doesn't know input length, and only relies on end-of-block properties */ + } + ip += length; op = cpy; + } else { + cpy = op+length; + if (endOnInput) { /* LZ4_decompress_safe() */ + DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); + /* We don't need to check oend, since we check it once for each loop below */ + if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; } + /* Literals can only be 14, but hope compilers optimize if we copy by a register size */ + memcpy(op, ip, 16); + } else { /* LZ4_decompress_fast() */ + /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : + * it doesn't know input length, and relies on end-of-block properties */ + memcpy(op, ip, 8); + if (length > 8) { memcpy(op+8, ip+8, 8); } + } + ip += length; op = cpy; } - else - { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + assert(match <= op); + + /* get matchlength */ + length = token & ML_MASK; + + if (length == ML_MASK) { + variable_length_error error = ok; + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ + length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); + if (error != ok) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + } else { + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + + /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */ + if ((dict == withPrefix64k) || (match >= lowPrefix)) { + if (offset >= 8) { + assert(match >= lowPrefix); + assert(match <= op); + assert(op + 18 <= oend); + + memcpy(op, match, 8); + memcpy(op+8, match+8, 8); + memcpy(op+16, match+16, 2); + op += length; + continue; + } } } + + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) { + length = MIN(length, (size_t)(oend-op)); /* reach end of buffer */ + } else { + goto _output_error; /* end-of-block condition violated */ + } } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) { *op++ = *copyFrom++; } + } else { + memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; } - memcpy(op, ip, length); - ip += length; - op += length; - break; /* Necessarily EOF, due to parsing restrictions */ - } - LZ4_WILDCOPY(op, ip, cpy); ip -= (op-cpy); op = cpy; - /* get offset */ - LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2; - if ((checkOffset) && (unlikely(ref < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */ + /* copy match within block */ + cpy = op + length; - /* get matchlength */ - if ((length=(token&ML_MASK)) == ML_MASK) - { - unsigned s; - do - { - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - s = *ip++; - length += s; - } while (s==255); - if ((safeDecode) && LZ4_32BITS && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */ + assert((op <= oend) && (oend-op >= 32)); + if (unlikely(offset<16)) { + LZ4_memcpy_using_offset(op, match, cpy, offset); + } else { + LZ4_wildCopy32(op, match, cpy); + } + + op = cpy; /* wildcopy correction */ } - length += MINMATCH; + safe_decode: +#endif - /* check external dictionary */ - if ((dict==usingExtDict) && (ref < (BYTE* const)dest)) - { - if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; + /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ + while (1) { + token = *ip++; + length = token >> ML_BITS; /* literal length */ + + assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ + + /* A two-stage shortcut for the most common case: + * 1) If the literal length is 0..14, and there is enough space, + * enter the shortcut and copy 16 bytes on behalf of the literals + * (in the fast mode, only 8 bytes can be safely copied this way). + * 2) Further if the match length is 4..18, copy 18 bytes in a similar + * manner; but we ensure that there's enough space in the output for + * those 18 bytes earlier, upon entering the shortcut (in other words, + * there is a combined check for both stages). + */ + if ( (endOnInput ? length != RUN_MASK : length <= 8) + /* strictly "less than" on input, to re-enter the loop with at least one byte */ + && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) { + /* Copy the literals */ + memcpy(op, ip, endOnInput ? 16 : 8); + op += length; ip += length; + + /* The second stage: prepare for match copying, decode full info. + * If it doesn't work out, the info won't be wasted. */ + length = token & ML_MASK; /* match length */ + offset = LZ4_readLE16(ip); ip += 2; + match = op - offset; + assert(match <= op); /* check overflow */ + + /* Do not deal with overlapping matches. */ + if ( (length != ML_MASK) + && (offset >= 8) + && (dict==withPrefix64k || match >= lowPrefix) ) { + /* Copy the match. */ + memcpy(op + 0, match + 0, 8); + memcpy(op + 8, match + 8, 8); + memcpy(op +16, match +16, 2); + op += length + MINMATCH; + /* Both stages worked, load the next token. */ + continue; + } + + /* The second stage didn't work out, but the info is ready. + * Propel it right to the point of match copying. */ + goto _copy_match; + } + + /* decode literal length */ + if (length == RUN_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); + if (error == initial_error) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + } - if (length <= (size_t)(dest-(char*)ref)) + /* copy literals */ + cpy = op+length; +#if LZ4_FAST_DEC_LOOP + safe_literal_copy: +#endif + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) { - ref = dictEnd - (dest-(char*)ref); - memcpy(op, ref, length); + /* We've either hit the input parsing restriction or the output parsing restriction. + * If we've hit the input parsing condition then this must be the last sequence. + * If we've hit the output parsing condition then we are either using partialDecoding + * or we've hit the output parsing condition. + */ + if (partialDecoding) { + /* Since we are partial decoding we may be in this block because of the output parsing + * restriction, which is not valid since the output buffer is allowed to be undersized. + */ + assert(endOnInput); + /* If we're in this block because of the input parsing condition, then we must be on the + * last sequence (or invalid), so we must check that we exactly consume the input. + */ + if ((ip+length>iend-(2+1+LASTLITERALS)) && (ip+length != iend)) { goto _output_error; } + assert(ip+length <= iend); + /* We are finishing in the middle of a literals segment. + * Break after the copy. + */ + if (cpy > oend) { + cpy = oend; + assert(op<=oend); + length = (size_t)(oend-op); + } + assert(ip+length <= iend); + } else { + /* We must be on the last sequence because of the parsing limitations so check + * that we exactly regenerate the original size (must be exact when !endOnInput). + */ + if ((!endOnInput) && (cpy != oend)) { goto _output_error; } + /* We must be on the last sequence (or invalid) because of the parsing limitations + * so check that we exactly consume the input and don't overrun the output buffer. + */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { goto _output_error; } + } + memmove(op, ip, length); /* supports overlapping memory regions, which only matters for in-place decompression scenarios */ + ip += length; op += length; + /* Necessarily EOF when !partialDecoding. When partialDecoding + * it is EOF if we've either filled the output buffer or hit + * the input parsing restriction. + */ + if (!partialDecoding || (cpy == oend) || (ip == iend)) { + break; + } + } else { + LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ + ip += length; op = cpy; } - else - { - size_t copySize = (size_t)(dest-(char*)ref); - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - copySize = length - copySize; - if (copySize > (size_t)((char*)op-dest)) /* overlap */ - { - BYTE* const endOfMatch = op + copySize; - const BYTE* copyFrom = (BYTE*)dest; - while (op < endOfMatch) *op++ = *copyFrom++; + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + + /* get matchlength */ + length = token & ML_MASK; + + _copy_match: + if (length == ML_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); + if (error != ok) goto _output_error; + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + +#if LZ4_FAST_DEC_LOOP + safe_match_copy: +#endif + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) length = MIN(length, (size_t)(oend-op)); + else goto _output_error; /* doesn't respect parsing restriction */ } - else - { - memcpy(op, dest, copySize); + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + memcpy(op, dictEnd - copySize, copySize); op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + assert(match >= lowPrefix); + + /* copy match within block */ + cpy = op + length; + + /* partialDecoding : may end anywhere within the block */ + assert(op<=oend); + if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + size_t const mlen = MIN(length, (size_t)(oend-op)); + const BYTE* const matchEnd = match + mlen; + BYTE* const copyEnd = op + mlen; + if (matchEnd > op) { /* overlap copy */ + while (op < copyEnd) { *op++ = *match++; } + } else { + memcpy(op, match, mlen); } + op = copyEnd; + if (op == oend) { break; } + continue; } - continue; - } - /* copy repeated sequence */ - cpy = op + length; - if (unlikely((op-ref)<(int)STEPSIZE)) - { - const size_t dec64 = dec64table[op-ref]; - op[0] = ref[0]; - op[1] = ref[1]; - op[2] = ref[2]; - op[3] = ref[3]; - ref += dec32table[op-ref]; - A32(op+4) = A32(ref); - op += 8; ref -= dec64; - } else { LZ4_COPY8(op,ref); } - - if (unlikely(cpy>oend-12)) - { - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last 5 bytes must be literals */ - if (op oend-MATCH_SAFEGUARD_DISTANCE)) { + BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy8(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op < cpy) { *op++ = *match++; } + } else { + memcpy(op, match, 8); + if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); } + } + op = cpy; /* wildcopy correction */ } - else LZ4_WILDCOPY(op, ref, cpy); - op=cpy; /* correction */ - } - /* end of decoding */ - if (endOnInput) - return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ - else - return (int) (((char*)ip)-source); /* Nb of input bytes read */ + /* end of decoding */ + if (endOnInput) { + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ + } else { + return (int) (((const char*)ip)-src); /* Nb of input bytes read */ + } - /* Overflow error detected */ -_output_error: - return (int) (-(((char*)ip)-source))-1; + /* Overflow error detected */ + _output_error: + return (int) (-(((const char*)ip)-src))-1; + } } +/*===== Instantiate the API decoding functions. =====*/ + +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, + endOnInputSize, decode_full_block, noDict, + (BYTE*)dest, NULL, 0); } -int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) +LZ4_FORCE_O2_GCC_PPC64LE +int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, int targetOutputSize, int dstCapacity) { - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, NULL, 0); + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, + endOnInputSize, partial_decode, + noDict, (BYTE*)dst, NULL, 0); } +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_fast(const char* source, char* dest, int originalSize) { - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 64 KB); + return LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, decode_full_block, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); } -/* streaming decompression functions */ +/*===== Instantiate a few more decoding cases, used more than once. =====*/ -typedef struct +LZ4_FORCE_O2_GCC_PPC64LE /* Exported, an obsolete API function. */ +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) { - const char* dictionary; - int dictSize; -} LZ4_streamDecode_t_internal; + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, decode_full_block, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} -/* - * If you prefer dynamic allocation methods, - * LZ4_createStreamDecode() - * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure. +/* Another obsolete API function, paired with the previous one. */ +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + /* LZ4_decompress_fast doesn't validate match offsets, + * and thus serves well with any prefixed dictionary. */ + return LZ4_decompress_fast(source, dest, originalSize); +} + +LZ4_FORCE_O2_GCC_PPC64LE +static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, decode_full_block, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2_GCC_PPC64LE +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, decode_full_block, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2_GCC_PPC64LE +static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize, + const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, decode_full_block, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +/* The "double dictionary" mode, for use with e.g. ring buffers: the first part + * of the dictionary is passed as prefix, and the second via dictStart + dictSize. + * These routines are used only once, in LZ4_decompress_*_continue(). */ +LZ4_FORCE_INLINE +int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize, const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, decode_full_block, usingExtDict, + (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_INLINE +int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalSize, + size_t prefixSize, const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, decode_full_block, usingExtDict, + (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); +} + +/*===== streaming decompression functions =====*/ + LZ4_streamDecode_t* LZ4_createStreamDecode(void) { - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(sizeof(U32), LZ4_STREAMDECODESIZE_U32); - MEM_INIT(lz4s, 0, LZ4_STREAMDECODESIZE); + LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); + LZ4_STATIC_ASSERT(LZ4_STREAMDECODESIZE >= sizeof(LZ4_streamDecode_t_internal)); /* A compilation error here means LZ4_STREAMDECODESIZE is not large enough */ return lz4s; } int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) { + if (LZ4_stream == NULL) { return 0; } /* support free on NULL */ FREEMEM(LZ4_stream); return 0; } -/* - * LZ4_setStreamDecode - * Use this function to instruct where to find the dictionary - * This function is not necessary if previous data is still available where it was decoded. - * Loading a size of 0 is allowed (same effect as no dictionary). - * Return : 1 if OK, 0 if error +/*! LZ4_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * This function is not necessary if previous data is still available where it was decoded. + * Loading a size of 0 is allowed (same effect as no dictionary). + * @return : 1 if OK, 0 if error */ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) { - LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; - lz4sd->dictionary = dictionary; - lz4sd->dictSize = dictSize; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + lz4sd->prefixSize = (size_t) dictSize; + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + lz4sd->externalDict = NULL; + lz4sd->extDictSize = 0; return 1; } +/*! LZ4_decoderRingBufferSize() : + * when setting a ring buffer for streaming decompression (optional scenario), + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * Note : in a ring buffer scenario, + * blocks are presumed decompressed next to each other. + * When not enough space remains for next block (remainingSize < maxBlockSize), + * decoding resumes from beginning of ring buffer. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +int LZ4_decoderRingBufferSize(int maxBlockSize) +{ + if (maxBlockSize < 0) return 0; + if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0; + if (maxBlockSize < 16) maxBlockSize = 16; + return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize); +} + /* *_continue() : These decoding functions allow decompression of multiple blocks in "streaming" mode. Previously decoded blocks must still be available at the memory position where they were decoded. If it's not possible, save the relevant part of decoded data into a safe buffer, - and indicate where it stands using LZ4_setDictDecode() + and indicate where it stands using LZ4_setStreamDecode() */ +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) { - LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize); - if (result <= 0) return result; - if (lz4sd->dictionary + lz4sd->dictSize == dest) - { - lz4sd->dictSize += result; - } - else - { - lz4sd->dictionary = dest; - lz4sd->dictSize = result; + if (lz4sd->prefixSize == 0) { + /* The first call, no dictionary yet. */ + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + /* They're rolling the current segment. */ + if (lz4sd->prefixSize >= 64 KB - 1) + result = LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + else if (lz4sd->extDictSize == 0) + result = LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize); + else + result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)result; + lz4sd->prefixEnd += result; + } else { + /* The buffer wraps around, or they're switching to another buffer. */ + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE*)dest + result; } return result; } +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) { - LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; - - result = LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize); - if (result <= 0) return result; - if (lz4sd->dictionary + lz4sd->dictSize == dest) - { - lz4sd->dictSize += result; - } - else - { - lz4sd->dictionary = dest; - lz4sd->dictSize = result; + assert(originalSize >= 0); + + if (lz4sd->prefixSize == 0) { + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_fast(source, dest, originalSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + if (lz4sd->prefixSize >= 64 KB - 1 || lz4sd->extDictSize == 0) + result = LZ4_decompress_fast(source, dest, originalSize); + else + result = LZ4_decompress_fast_doubleDict(source, dest, originalSize, + lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)originalSize; + lz4sd->prefixEnd += originalSize; + } else { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_fast_extDict(source, dest, originalSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; } return result; @@ -1154,114 +2304,95 @@ Advanced decoding functions : the dictionary must be explicitly provided within parameters */ -FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { if (dictSize==0) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, NULL, 64 KB); - if ((dictStart+dictSize == dest) && (dictSize >= (int)(64 KB - 1))) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, NULL, 64 KB); - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, dictStart, dictSize); + return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); + if (dictStart+dictSize == dest) { + if (dictSize >= 64 KB - 1) { + return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize); } -int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { - //return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, dictStart, dictSize); - return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); + if (dictSize==0 || dictStart+dictSize == dest) + return LZ4_decompress_fast(source, dest, originalSize); + assert(dictSize >= 0); + return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize); } -int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) + +/*=************************************************* +* Obsolete Functions +***************************************************/ +/* obsolete compression functions */ +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { - //return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, dictStart, dictSize); - return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); + return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } - -/* debug function */ -int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +int LZ4_compress(const char* src, char* dest, int srcSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, dictStart, dictSize); + return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize)); +} +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); +} +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); +} +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity) +{ + return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1); +} +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) +{ + return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } - -/*************************************************** - Obsolete Functions -***************************************************/ /* -These function names are deprecated and should no longer be used. +These decompression functions are deprecated and should no longer be used. They are only provided here for compatibility with older user programs. - LZ4_uncompress is totally equivalent to LZ4_decompress_fast - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe */ -int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } -int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } - - -/* Obsolete Streaming functions */ - -int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } - -static void LZ4_init(LZ4_stream_t_internal* lz4ds, const BYTE* base) +int LZ4_uncompress (const char* source, char* dest, int outputSize) { - MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE); - lz4ds->bufferStart = base; + return LZ4_decompress_fast(source, dest, outputSize); } - -int LZ4_resetStreamState(void* state, const char* inputBuffer) -{ - if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t_internal*)state, (const BYTE*)inputBuffer); - return 0; -} - -void* LZ4_create (const char* inputBuffer) +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { - void* lz4ds = ALLOCATOR(4, LZ4_STREAMSIZE_U32); - LZ4_init ((LZ4_stream_t_internal*)lz4ds, (const BYTE*)inputBuffer); - return lz4ds; + return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } -char* LZ4_slideInputBuffer (void* LZ4_Data) -{ - LZ4_stream_t_internal* lz4ds = (LZ4_stream_t_internal*)LZ4_Data; - - LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)lz4ds->bufferStart, 64 KB); - - return (char*)(lz4ds->bufferStart + 64 KB); -} - -/* Obsolete compresson functions using User-allocated state */ +/* Obsolete Streaming functions */ -int LZ4_sizeofState() { return LZ4_STREAMSIZE; } +int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } -int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize) +int LZ4_resetStreamState(void* state, char* inputBuffer) { - if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */ - MEM_INIT(state, 0, LZ4_STREAMSIZE); - - if (inputSize < (int)LZ4_64KLIMIT) - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue); - else - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64BITS ? byU32 : byPtr, noDict, noDictIssue); + (void)inputBuffer; + LZ4_resetStream((LZ4_stream_t*)state); + return 0; } -int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) +void* LZ4_create (char* inputBuffer) { - if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */ - MEM_INIT(state, 0, LZ4_STREAMSIZE); - - if (inputSize < (int)LZ4_64KLIMIT) - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue); - else - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64BITS ? byU32 : byPtr, noDict, noDictIssue); + (void)inputBuffer; + return LZ4_createStream(); } -/* Obsolete streaming decompression functions */ - -int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +char* LZ4_slideInputBuffer (void* state) { - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, NULL, 64 KB); + /* avoid const char * -> char * conversion warning */ + return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary; } -int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) -{ - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 64 KB); -} +#endif /* LZ4_COMMONDEFS_ONLY */ diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.h index 44ada149e1240..32108e2329189 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.h @@ -1,7 +1,8 @@ /* - LZ4 - Fast LZ compression algorithm - Header File - Copyright (C) 2011-2014, Yann Collet. + * LZ4 - Fast LZ compression algorithm + * Header File + * Copyright (C) 2011-present, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -28,294 +29,734 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 */ -#pragma once - #if defined (__cplusplus) extern "C" { #endif +#ifndef LZ4_H_2983827168210 +#define LZ4_H_2983827168210 + +/* --- Dependency --- */ +#include /* size_t */ + + +/** + Introduction + + LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core, + scalable with multi-cores CPU. It features an extremely fast decoder, with speed in + multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. + + The LZ4 compression library provides in-memory compression and decompression functions. + It gives full buffer control to user. + Compression can be done in: + - a single step (described as Simple Functions) + - a single step, reusing a context (described in Advanced Functions) + - unbounded multiple steps (described as Streaming compression) + + lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). + Decompressing such a compressed block requires additional metadata. + Exact metadata depends on exact decompression function. + For the typical case of LZ4_decompress_safe(), + metadata includes block's compressed size, and maximum bound of decompressed size. + Each application is free to encode and pass such metadata in whichever way it wants. + + lz4.h only handle blocks, it can not generate Frames. + + Blocks are different from Frames (doc/lz4_Frame_format.md). + Frames bundle both blocks and metadata in a specified manner. + Embedding metadata is required for compressed data to be self-contained and portable. + Frame format is delivered through a companion API, declared in lz4frame.h. + The `lz4` CLI can only manage frames. +*/ + +/*^*************************************************************** +* Export parameters +*****************************************************************/ /* - * lz4.h provides raw compression format functions, for optimal performance and integration into programs. - * If you need to generate data using an inter-operable format (respecting the framing specification), - * please use lz4frame.h instead. +* LZ4_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +* LZ4LIB_VISIBILITY : +* Control library symbols visibility. */ +#ifndef LZ4LIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define LZ4LIB_VISIBILITY +# endif +#endif +#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) +# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY +#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) +# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define LZ4LIB_API LZ4LIB_VISIBILITY +#endif + +/*------ Version ------*/ +#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ -/************************************** - Version -**************************************/ -#define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */ -#define LZ4_VERSION_MINOR 3 /* for minor interface/format changes */ -#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) -int LZ4_versionNumber (void); -/************************************** - Tuning parameter +#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE +#define LZ4_QUOTE(str) #str +#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) +#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) + +LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version */ +LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version */ + + +/*-************************************ +* Tuning parameter **************************************/ -/* +/*! * LZ4_MEMORY_USAGE : * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio - * Reduced memory usage can improve speed, due to cache effect + * Increasing memory usage improves compression ratio. + * Reduced memory usage may improve speed, thanks to better cache locality. * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#define LZ4_MEMORY_USAGE 14 +#ifndef LZ4_MEMORY_USAGE +# define LZ4_MEMORY_USAGE 14 +#endif -/************************************** - Simple Functions +/*-************************************ +* Simple Functions **************************************/ - -int LZ4_compress (const char* source, char* dest, int sourceSize); -int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); - -/* -LZ4_compress() : - Compresses 'sourceSize' bytes from 'source' into 'dest'. - Destination buffer must be already allocated, - and must be sized to handle worst cases situations (input data not compressible) - Worst case size evaluation is provided by function LZ4_compressBound() - inputSize : Max supported value is LZ4_MAX_INPUT_SIZE - return : the number of bytes written in buffer dest - or 0 if the compression fails - -LZ4_decompress_safe() : - compressedSize : is obviously the source size - maxDecompressedSize : is the size of the destination buffer, which must be already allocated. - return : the number of bytes decompressed into the destination buffer (necessarily <= maxDecompressedSize) - If the destination buffer is not large enough, decoding will stop and output an error code (<0). - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against buffer overflow exploits, - and never writes outside of output buffer, nor reads outside of input buffer. - It is also protected against malicious data packets. -*/ +/*! LZ4_compress_default() : + * Compresses 'srcSize' bytes from buffer 'src' + * into already allocated 'dst' buffer of size 'dstCapacity'. + * Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). + * It also runs faster, so it's a recommended setting. + * If the function cannot compress 'src' into a more limited 'dst' budget, + * compression stops *immediately*, and the function result is zero. + * In which case, 'dst' content is undefined (invalid). + * srcSize : max supported value is LZ4_MAX_INPUT_SIZE. + * dstCapacity : size of buffer 'dst' (which must be already allocated) + * @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + * or 0 if compression fails + * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + */ +LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); + +/*! LZ4_decompress_safe() : + * compressedSize : is the exact complete size of the compressed block. + * dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size. + * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * Note 1 : This function is protected against malicious data packets : + * it will never writes outside 'dst' buffer, nor read outside 'source' buffer, + * even if the compressed block is maliciously modified to order the decoder to do these actions. + * In such case, the decoder stops immediately, and considers the compressed block malformed. + * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them. + * The implementation is free to send / store / derive this information in whichever way is most beneficial. + * If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead. + */ +LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); -/************************************** - Advanced Functions +/*-************************************ +* Advanced Functions **************************************/ #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ -#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) +#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) -/* -LZ4_compressBound() : +/*! LZ4_compressBound() : Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) - This function is primarily useful for memory allocation purposes (output buffer size). + This function is primarily useful for memory allocation purposes (destination buffer size). Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - - isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE - return : maximum output size in a "worst case" scenario - or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) + Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize) + inputSize : max supported value is LZ4_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is incorrect (too large or negative) */ -int LZ4_compressBound(int isize); - - -/* -LZ4_compress_limitedOutput() : - Compress 'sourceSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. - If it cannot achieve it, compression will stop, and result of the function will be zero. - This saves time and memory on detecting non-compressible (or barely compressible) data. - This function never writes outside of provided output buffer. - - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxOutputSize : is the size of the destination buffer (which must be already allocated) - return : the number of bytes written in buffer 'dest' - or 0 if compression fails +LZ4LIB_API int LZ4_compressBound(int inputSize); + +/*! LZ4_compress_fast() : + Same as LZ4_compress_default(), but allows selection of "acceleration" factor. + The larger the acceleration value, the faster the algorithm, but also the lesser the compression. + It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. + An acceleration value of "1" is the same as regular LZ4_compress_default() + Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c). */ -int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); -/* -LZ4_compress_withState() : - Same compression functions, but using an externally allocated memory space to store compression state. - Use LZ4_sizeofState() to know how much memory must be allocated, - and then, provide it as 'void* state' to compression functions. +/*! LZ4_compress_fast_extState() : + * Same as LZ4_compress_fast(), using an externally allocated memory space for its state. + * Use LZ4_sizeofState() to know how much memory must be allocated, + * and allocate it on 8-bytes boundaries (using `malloc()` typically). + * Then, provide this buffer as `void* state` to compression function. + */ +LZ4LIB_API int LZ4_sizeofState(void); +LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + + +/*! LZ4_compress_destSize() : + * Reverse the logic : compresses as much data as possible from 'src' buffer + * into already allocated buffer 'dst', of size >= 'targetDestSize'. + * This function either compresses the entire 'src' content into 'dst' if it's large enough, + * or fill 'dst' buffer completely with as much data as possible from 'src'. + * note: acceleration parameter is fixed to "default". + * + * *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'. + * New value is necessarily <= input value. + * @return : Nb bytes written into 'dst' (necessarily <= targetDestSize) + * or 0 if compression fails. */ -int LZ4_sizeofState(void); -int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); + + +/*! LZ4_decompress_safe_partial() : + * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', + * into destination buffer 'dst' of size 'dstCapacity'. + * Up to 'targetOutputSize' bytes will be decoded. + * The function stops decoding on reaching this objective, + * which can boost performance when only the beginning of a block is required. + * + * @return : the number of bytes decoded in `dst` (necessarily <= dstCapacity) + * If source stream is detected malformed, function returns a negative result. + * + * Note : @return can be < targetOutputSize, if compressed block contains less data. + * + * Note 2 : this function features 2 parameters, targetOutputSize and dstCapacity, + * and expects targetOutputSize <= dstCapacity. + * It effectively stops decoding on reaching targetOutputSize, + * so dstCapacity is kind of redundant. + * This is because in a previous version of this function, + * decoding operation would not "break" a sequence in the middle. + * As a consequence, there was no guarantee that decoding would stop at exactly targetOutputSize, + * it could write more bytes, though only up to dstCapacity. + * Some "margin" used to be required for this operation to work properly. + * This is no longer necessary. + * The function nonetheless keeps its signature, in an effort to not break API. + */ +LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); -/* -LZ4_decompress_fast() : - originalSize : is the original and therefore uncompressed size - return : the number of bytes read from the source buffer (in other words, the compressed size) - If the source stream is detected malformed, the function will stop decoding and return a negative result. - Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. - note : This function fully respect memory boundaries for properly formed compressed data. - It is a bit faster than LZ4_decompress_safe(). - However, it does not provide any protection against intentionally modified data stream (malicious input). - Use this function in trusted environment only (data to decode comes from a trusted source). -*/ -int LZ4_decompress_fast (const char* source, char* dest, int originalSize); +/*-********************************************* +* Streaming Compression Functions +***********************************************/ +typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ + +LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); +LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); + +/*! LZ4_resetStream_fast() : v1.9.0+ + * Use this to prepare an LZ4_stream_t for a new chain of dependent blocks + * (e.g., LZ4_compress_fast_continue()). + * + * An LZ4_stream_t must be initialized once before usage. + * This is automatically done when created by LZ4_createStream(). + * However, should the LZ4_stream_t be simply declared on stack (for example), + * it's necessary to initialize it first, using LZ4_initStream(). + * + * After init, start any new stream with LZ4_resetStream_fast(). + * A same LZ4_stream_t can be re-used multiple times consecutively + * and compress multiple streams, + * provided that it starts each new stream with LZ4_resetStream_fast(). + * + * LZ4_resetStream_fast() is much faster than LZ4_initStream(), + * but is not compatible with memory regions containing garbage data. + * + * Note: it's only useful to call LZ4_resetStream_fast() + * in the context of streaming compression. + * The *extState* functions perform their own resets. + * Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive. + */ +LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); + +/*! LZ4_loadDict() : + * Use this function to reference a static dictionary into LZ4_stream_t. + * The dictionary must remain available during compression. + * LZ4_loadDict() triggers a reset, so any previous data will be forgotten. + * The same dictionary will have to be loaded on decompression side for successful decoding. + * Dictionary are useful for better compression of small data (KB range). + * While LZ4 accept any input as dictionary, + * results are generally better when using Zstandard's Dictionary Builder. + * Loading a size of 0 is allowed, and is the same as reset. + * @return : loaded dictionary size, in bytes (necessarily <= 64 KB) + */ +LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); + +/*! LZ4_compress_fast_continue() : + * Compress 'src' content using data from previously compressed blocks, for better compression ratio. + * 'dst' buffer must be already allocated. + * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * + * @return : size of compressed block + * or 0 if there is an error (typically, cannot fit into 'dst'). + * + * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. + * Each block has precise boundaries. + * Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata. + * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. + * + * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory ! + * + * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. + * Make sure that buffers are separated, by at least one byte. + * This construction ensures that each block only depends on previous block. + * + * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + * + * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed. + */ +LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_saveDict() : + * If last 64KB data cannot be guaranteed to remain available at its current memory location, + * save it into a safer place (char* safeBuffer). + * This is schematically equivalent to a memcpy() followed by LZ4_loadDict(), + * but is much faster, because LZ4_saveDict() doesn't need to rebuild tables. + * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error. + */ +LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize); -/* -LZ4_decompress_safe_partial() : - This function decompress a compressed block of size 'compressedSize' at position 'source' - into destination buffer 'dest' of size 'maxDecompressedSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. - Always control how many bytes were decoded. - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets +/*-********************************************** +* Streaming Decompression Functions +* Bufferless synchronous API +************************************************/ +typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */ + +/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : + * creation / destruction of streaming decompression tracking context. + * A tracking context can be re-used multiple times. + */ +LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); +LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); + +/*! LZ4_setStreamDecode() : + * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. + * Use this function to start decompression of a new stream of blocks. + * A dictionary can optionally be set. Use NULL or size 0 for a reset order. + * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. + * @return : 1 if OK, 0 if error + */ +LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); + +/*! LZ4_decoderRingBufferSize() : v1.8.2+ + * Note : in a ring buffer scenario (optional), + * blocks are presumed decompressed next to each other + * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), + * at which stage it resumes from beginning of ring buffer. + * When setting such a ring buffer for streaming decompression, + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); +#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ + +/*! LZ4_decompress_*_continue() : + * These decoding functions allow decompression of consecutive blocks in "streaming" mode. + * A block is an unsplittable entity, it must be presented entirely to a decompression function. + * Decompression functions only accepts one block at a time. + * The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded. + * If less than 64KB of data has been decoded, all the data must be present. + * + * Special : if decompression side sets a ring buffer, it must respect one of the following conditions : + * - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize). + * maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes. + * In which case, encoding and decoding buffers do not need to be synchronized. + * Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize. + * - Synchronized mode : + * Decompression buffer size is _exactly_ the same as compression buffer size, + * and follows exactly same update rule (block boundaries at same positions), + * and decoding function is provided with exact decompressed size of each block (exception for last block of the stream), + * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB). + * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes. + * In which case, encoding and decoding buffers do not need to be synchronized, + * and encoding ring buffer can have any size, including small ones ( < 64 KB). + * + * Whenever these conditions are not possible, + * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, + * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. */ -int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); +LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity); -/*********************************************** - Experimental Streaming Compression Functions -***********************************************/ - -#define LZ4_STREAMSIZE_U32 ((1 << (LZ4_MEMORY_USAGE-2)) + 8) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U32 * sizeof(unsigned int)) -/* - * LZ4_stream_t - * information structure to track an LZ4 stream. - * important : init this structure content before first use ! +/*! LZ4_decompress_*_usingDict() : + * These decoding functions work the same as + * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() + * They are stand-alone, and don't need an LZ4_streamDecode_t structure. + * Dictionary is presumed stable : it must remain accessible and unmodified during decompression. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. */ -typedef struct { unsigned int table[LZ4_STREAMSIZE_U32]; } LZ4_stream_t; +LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); + +#endif /* LZ4_H_2983827168210 */ + + +/*^************************************* + * !!!!!! STATIC LINKING ONLY !!!!!! + ***************************************/ + +/*-**************************************************************************** + * Experimental section + * + * Symbols declared in this section must be considered unstable. Their + * signatures or semantics may change, or they may be removed altogether in the + * future. They are therefore only safe to depend on when the caller is + * statically linked against the library. + * + * To protect against unsafe usage, not only are the declarations guarded, + * the definitions are hidden by default + * when building LZ4 as a shared/dynamic library. + * + * In order to access these declarations, + * define LZ4_STATIC_LINKING_ONLY in your application + * before including LZ4's headers. + * + * In order to make their implementations accessible dynamically, you must + * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. + ******************************************************************************/ + +#ifdef LZ4_STATIC_LINKING_ONLY + +#ifndef LZ4_STATIC_3504398509 +#define LZ4_STATIC_3504398509 + +#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS +#define LZ4LIB_STATIC_API LZ4LIB_API +#else +#define LZ4LIB_STATIC_API +#endif -/* - * LZ4_resetStream - * Use this function to init an allocated LZ4_stream_t structure - */ -void LZ4_resetStream (LZ4_stream_t* LZ4_streamPtr); -/* - * If you prefer dynamic allocation methods, - * LZ4_createStream will allocate and initialize an LZ4_stream_t structure - * LZ4_freeStream releases its memory. +/*! LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. + * It is only safe to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). + * From a high level, the difference is that + * this function initializes the provided state with a call to something like LZ4_resetStream_fast() + * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). */ -LZ4_stream_t* LZ4_createStream(void); -int LZ4_freeStream (LZ4_stream_t* LZ4_stream); - -/* - * LZ4_loadDict - * Use this function to load a static dictionary into LZ4_stream. - * Any previous data will be forgotten, only 'dictionary' will remain in memory. - * Loading a size of 0 is allowed. - * Return : 1 if OK, 0 if error +LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_attach_dictionary() : + * This is an experimental API that allows + * efficient use of a static dictionary many times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a + * working LZ4_stream_t, this function introduces a no-copy setup mechanism, + * in which the working stream references the dictionary stream in-place. + * + * Several assumptions are made about the state of the dictionary stream. + * Currently, only streams which have been prepared by LZ4_loadDict() should + * be expected to work. + * + * Alternatively, the provided dictionaryStream may be NULL, + * in which case any existing dictionary stream is unset. + * + * If a dictionary is provided, it replaces any pre-existing stream history. + * The dictionary contents are the only history that can be referenced and + * logically immediately precede the data compressed in the first subsequent + * compression call. + * + * The dictionary will only remain attached to the working stream through the + * first compression call, at the end of which it is cleared. The dictionary + * stream (and source buffer) must remain in-place / accessible / unchanged + * through the completion of the first compression call on the stream. */ -int LZ4_loadDict (LZ4_stream_t* LZ4_stream, const char* dictionary, int dictSize); - -/* - * LZ4_compress_continue - * Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio - * Previous data blocks are assumed to still be present at their previous location. +LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream); + + +/*! In-place compression and decompression + * + * It's possible to have input and output sharing the same buffer, + * for highly contrained memory environments. + * In both cases, it requires input to lay at the end of the buffer, + * and decompression to start at beginning of the buffer. + * Buffer size must feature some margin, hence be larger than final size. + * + * |<------------------------buffer--------------------------------->| + * |<-----------compressed data--------->| + * |<-----------decompressed size------------------>| + * |<----margin---->| + * + * This technique is more useful for decompression, + * since decompressed size is typically larger, + * and margin is short. + * + * In-place decompression will work inside any buffer + * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). + * This presumes that decompressedSize > compressedSize. + * Otherwise, it means compression actually expanded data, + * and it would be more efficient to store such data with a flag indicating it's not compressed. + * This can happen when data is not compressible (already compressed, or encrypted). + * + * For in-place compression, margin is larger, as it must be able to cope with both + * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, + * and data expansion, which can happen when input is not compressible. + * As a consequence, buffer size requirements are much higher, + * and memory savings offered by in-place compression are more limited. + * + * There are ways to limit this cost for compression : + * - Reduce history size, by modifying LZ4_DISTANCE_MAX. + * Note that it is a compile-time constant, so all compressions will apply this limit. + * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, + * so it's a reasonable trick when inputs are known to be small. + * - Require the compressor to deliver a "maximum compressed size". + * This is the `dstCapacity` parameter in `LZ4_compress*()`. + * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, + * in which case, the return code will be 0 (zero). + * The caller must be ready for these cases to happen, + * and typically design a backup scheme to send data uncompressed. + * The combination of both techniques can significantly reduce + * the amount of margin required for in-place compression. + * + * In-place compression can work in any buffer + * which size is >= (maxCompressedSize) + * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. + * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, + * so it's possible to reduce memory requirements by playing with them. */ -int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize); -/* - * LZ4_compress_limitedOutput_continue - * Same as before, but also specify a maximum target compressed size (maxOutputSize) - * If objective cannot be met, compression exits, and returns a zero. - */ -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize); +#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32) +#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */ -/* - * LZ4_saveDict - * If previously compressed data block is not guaranteed to remain available at its memory location - * save it into a safer place (char* safeBuffer) - * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call again LZ4_compress_continue() - * Return : dictionary size in bytes, or 0 if error - * Note : any dictSize > 64 KB will be interpreted as 64KB. - */ -int LZ4_saveDict (LZ4_stream_t* LZ4_stream, char* safeBuffer, int dictSize); +#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */ +# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ +#endif +#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */ +#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */ + +#endif /* LZ4_STATIC_3504398509 */ +#endif /* LZ4_STATIC_LINKING_ONLY */ + + + +#ifndef LZ4_H_98237428734687 +#define LZ4_H_98237428734687 + +/*-************************************************************ + * PRIVATE DEFINITIONS + ************************************************************** + * Do not use these definitions directly. + * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * Accessing members will expose code to API and/or ABI break in future versions of the library. + **************************************************************/ +#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) +#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) +#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ + +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#include + +typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; +struct LZ4_stream_t_internal { + uint32_t hashTable[LZ4_HASH_SIZE_U32]; + uint32_t currentOffset; + uint16_t dirty; + uint16_t tableType; + const uint8_t* dictionary; + const LZ4_stream_t_internal* dictCtx; + uint32_t dictSize; +}; + +typedef struct { + const uint8_t* externalDict; + size_t extDictSize; + const uint8_t* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#else + +typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; +struct LZ4_stream_t_internal { + unsigned int hashTable[LZ4_HASH_SIZE_U32]; + unsigned int currentOffset; + unsigned short dirty; + unsigned short tableType; + const unsigned char* dictionary; + const LZ4_stream_t_internal* dictCtx; + unsigned int dictSize; +}; + +typedef struct { + const unsigned char* externalDict; + const unsigned char* prefixEnd; + size_t extDictSize; + size_t prefixSize; +} LZ4_streamDecode_t_internal; -/************************************************ - Experimental Streaming Decompression Functions -************************************************/ +#endif -#define LZ4_STREAMDECODESIZE_U32 4 -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U32 * sizeof(unsigned int)) -/* - * LZ4_streamDecode_t - * information structure to track an LZ4 stream. - * important : init this structure content using LZ4_setStreamDecode or memset() before first use ! +/*! LZ4_stream_t : + * information structure to track an LZ4 stream. + * LZ4_stream_t can also be created using LZ4_createStream(), which is recommended. + * The structure definition can be convenient for static allocation + * (on stack, or as part of larger structure). + * Init this structure with LZ4_initStream() before first use. + * note : only use this definition in association with static linking ! + * this definition is not API/ABI safe, and may change in a future version. */ -typedef struct { unsigned int table[LZ4_STREAMDECODESIZE_U32]; } LZ4_streamDecode_t; - -/* - * LZ4_setStreamDecode - * Use this function to instruct where to find the dictionary. - * This function can be used to specify a static dictionary, - * or to instruct where to find some previously decoded data saved into a different memory space. - * Setting a size of 0 is allowed (same effect as no dictionary). - * Return : 1 if OK, 0 if error +#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4 + ((sizeof(void*)==16) ? 4 : 0) /*AS-400*/ ) +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) +union LZ4_stream_u { + unsigned long long table[LZ4_STREAMSIZE_U64]; + LZ4_stream_t_internal internal_donotuse; +} ; /* previously typedef'd to LZ4_stream_t */ + +/*! LZ4_initStream() : v1.9.0+ + * An LZ4_stream_t structure must be initialized at least once. + * This is automatically done when invoking LZ4_createStream(), + * but it's not when the structure is simply declared on stack (for example). + * + * Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t. + * It can also initialize any arbitrary buffer of sufficient size, + * and will @return a pointer of proper type upon initialization. + * + * Note : initialization fails if size and alignment conditions are not respected. + * In which case, the function will @return NULL. + * Note2: An LZ4_stream_t structure guarantees correct alignment and size. + * Note3: Before v1.9.0, use LZ4_resetStream() instead */ -int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); +LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size); -/* - * If you prefer dynamic allocation methods, - * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure - * LZ4_freeStreamDecode releases its memory. + +/*! LZ4_streamDecode_t : + * information structure to track an LZ4 stream during decompression. + * init this structure using LZ4_setStreamDecode() before first use. + * note : only use in association with static linking ! + * this definition is not API/ABI safe, + * and may change in a future version ! */ -LZ4_streamDecode_t* LZ4_createStreamDecode(void); -int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); +#define LZ4_STREAMDECODESIZE_U64 (4 + ((sizeof(void*)==16) ? 2 : 0) /*AS-400*/ ) +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) +union LZ4_streamDecode_u { + unsigned long long table[LZ4_STREAMDECODESIZE_U64]; + LZ4_streamDecode_t_internal internal_donotuse; +} ; /* previously typedef'd to LZ4_streamDecode_t */ -/* -*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks must still be available at the memory position where they were decoded. - If it's not possible, save the relevant part of decoded data into a safe buffer, - and indicate where its new address using LZ4_setStreamDecode() -*/ -int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); -int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); -/* -Advanced decoding functions : -*_usingDict() : - These decoding functions work the same as - a combination of LZ4_setDictDecode() followed by LZ4_decompress_x_continue() - They don't use nor update an LZ4_streamDecode_t structure. -*/ -int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); -int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); - +/*-************************************ +* Obsolete Functions +**************************************/ +/*! Deprecation warnings + * + * Deprecated functions make the compiler generate a warning when invoked. + * This is meant to invite users to update their source code. + * Should deprecation warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc + * or _CRT_SECURE_NO_WARNINGS in Visual. + * + * Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS + * before including the header file. + */ +#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS +# define LZ4_DEPRECATED(message) /* disable deprecation warnings */ +#else +# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define LZ4_DEPRECATED(message) [[deprecated(message)]] +# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif (LZ4_GCC_VERSION >= 301) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") +# define LZ4_DEPRECATED(message) +# endif +#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ + +/* Obsolete compression functions */ +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/* Obsolete decompression functions */ +LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize); +LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); + +/* Obsolete streaming functions; degraded functionality; do not use! + * + * In order to perform streaming compression, these functions depended on data + * that is no longer tracked in the state. They have been preserved as well as + * possible: using them will still produce a correct output. However, they don't + * actually retain any history between compression calls. The compression ratio + * achieved will therefore be no better than compressing each chunk + * independently. + */ +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void); +LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); -/************************************** - Obsolete Functions -**************************************/ -/* -Obsolete decompression functions -These function names are deprecated and should no longer be used. -They are only provided here for compatibility with older user programs. -- LZ4_uncompress is the same as LZ4_decompress_fast -- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe -These function prototypes are now disabled; uncomment them if you really need them. -It is highly recommended to stop using these functions and migrated to newer ones */ -/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */ -/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */ +/* Obsolete streaming decoding functions */ +LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); + +/*! LZ4_decompress_fast() : **unsafe!** + * These functions used to be faster than LZ4_decompress_safe(), + * but it has changed, and they are now slower than LZ4_decompress_safe(). + * This is because LZ4_decompress_fast() doesn't know the input size, + * and therefore must progress more cautiously in the input buffer to not read beyond the end of block. + * On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability. + * As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated. + * + * The last remaining LZ4_decompress_fast() specificity is that + * it can decompress a block without knowing its compressed size. + * Such functionality could be achieved in a more secure manner, + * by also providing the maximum size of input buffer, + * but it would require new prototypes, and adaptation of the implementation to this new use case. + * + * Parameters: + * originalSize : is the uncompressed size to regenerate. + * `dst` must be already allocated, its size must be >= 'originalSize' bytes. + * @return : number of bytes read from source buffer (== compressed size). + * The function expects to finish at block's end exactly. + * If the source stream is detected malformed, the function stops decoding and returns a negative result. + * note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer. + * However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds. + * Also, since match offsets are not validated, match reads from 'src' may underflow too. + * These issues never happen if input (compressed) data is correct. + * But they may happen if input data is invalid (error or intentional tampering). + * As a consequence, use these functions in trusted environments with trusted data **only**. + */ -/* - * If you prefer dynamic allocation methods, - * LZ4_createStreamDecode() - * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure. - * LZ4_free just frees it. +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead") +LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead") +LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead") +LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); + +/*! LZ4_resetStream() : + * An LZ4_stream_t structure must be initialized at least once. + * This is done with LZ4_initStream(), or LZ4_resetStream(). + * Consider switching to LZ4_initStream(), + * invoking LZ4_resetStream() will trigger deprecation warnings in the future. */ -/* void* LZ4_createStreamDecode(void); */ -/*int LZ4_free (void* LZ4_stream); yes, it's the same one as for compression */ +LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); -/* Obsolete streaming functions; use new streaming interface whenever possible */ -void* LZ4_create (const char* inputBuffer); -int LZ4_sizeofStreamState(void); -int LZ4_resetStreamState(void* state, const char* inputBuffer); -char* LZ4_slideInputBuffer (void* state); -/* Obsolete streaming decoding functions */ -int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compressedSize, int maxOutputSize); -int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize); +#endif /* LZ4_H_98237428734687 */ #if defined (__cplusplus) diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.c index 34a61733f258e..5922ed7b16b18 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.c @@ -1,604 +1,526 @@ /* - LZ4 HC - High Compression Mode of LZ4 - Copyright (C) 2011-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ + LZ4 HC - High Compression Mode of LZ4 + Copyright (C) 2011-2017, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ +/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ +/* ************************************* +* Tuning Parameter +***************************************/ -/************************************** - Tuning Parameter -**************************************/ -#define LZ4HC_DEFAULT_COMPRESSIONLEVEL 8 - - -/************************************** - Memory routines -**************************************/ -#include /* calloc, free */ -#define ALLOCATOR(s) calloc(1,s) -#define FREEMEM free -#include /* memset, memcpy */ -#define MEM_INIT memset - - -/************************************** - CPU Feature Detection -**************************************/ -/* 32 or 64 bits ? */ -#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ - || defined(__64BIT__) || defined(__mips64) \ - || defined(__powerpc64__) || defined(__powerpc64le__) \ - || defined(__ppc64__) || defined(__ppc64le__) \ - || defined(__PPC64__) || defined(__PPC64LE__) \ - || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) \ - || defined(__s390x__) ) /* Detects 64 bits mode */ -# define LZ4_ARCH64 1 -#else -# define LZ4_ARCH64 0 -#endif - -/* - * Little Endian or Big Endian ? - * Overwrite the #define below if you know your architecture endianess - */ -#include /* Apparently required to detect endianess */ -#if defined (__GLIBC__) -# include -# if (__BYTE_ORDER == __BIG_ENDIAN) -# define LZ4_BIG_ENDIAN 1 -# endif -#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN)) -# define LZ4_BIG_ENDIAN 1 -#elif defined(__sparc) || defined(__sparc__) \ - || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \ - || defined(__hpux) || defined(__hppa) \ - || defined(_MIPSEB) || defined(__s390__) -# define LZ4_BIG_ENDIAN 1 -#else -/* Little Endian assumed. PDP Endian and other very rare endian format are unsupported. */ -#endif - -/* - * Unaligned memory access is automatically enabled for "common" CPU, such as x86. - * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected - * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance +/*! HEAPMODE : + * Select how default compression function will allocate workplace memory, + * in stack (0:fastest), or in heap (1:requires malloc()). + * Since workplace is rather large, heap mode is recommended. */ -#if defined(__ARM_FEATURE_UNALIGNED) -# define LZ4_FORCE_UNALIGNED_ACCESS 1 +#ifndef LZ4HC_HEAPMODE +# define LZ4HC_HEAPMODE 1 #endif -/* Define this parameter if your target system or compiler does not support hardware bit count */ -#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ -# define LZ4_FORCE_SW_BITCOUNT -#endif - - -/************************************** - Compiler Options -**************************************/ -#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -/* "restrict" is a known keyword */ -#else -# define restrict /* Disable restrict */ -#endif -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include /* For Visual 2005 */ -# if LZ4_ARCH64 /* 64-bits */ -# pragma intrinsic(_BitScanForward64) /* For Visual 2005 */ -# pragma intrinsic(_BitScanReverse64) /* For Visual 2005 */ -# else /* 32-bits */ -# pragma intrinsic(_BitScanForward) /* For Visual 2005 */ -# pragma intrinsic(_BitScanReverse) /* For Visual 2005 */ -# endif -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4701) /* disable: C4701: potentially uninitialized local variable used */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - -#ifdef _MSC_VER /* Visual Studio */ -# define lz4_bswap16(x) _byteswap_ushort(x) -#else -# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8))) -#endif - - -/************************************** - Includes -**************************************/ +/*=== Dependency ===*/ +#define LZ4_HC_STATIC_LINKING_ONLY #include "lz4hc.h" -#include "lz4.h" -/************************************** - Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; +/*=== Common LZ4 definitions ===*/ +#if defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wunused-function" #endif - -#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) -#else -# define _PACKED +#if defined (__clang__) +# pragma clang diagnostic ignored "-Wunused-function" #endif -#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# ifdef __IBMC__ -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif -#endif +/*=== Enums ===*/ +typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive; -typedef struct _U16_S { U16 v; } _PACKED U16_S; -typedef struct _U32_S { U32 v; } _PACKED U32_S; -typedef struct _U64_S { U64 v; } _PACKED U64_S; -#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# pragma pack(pop) +#define LZ4_COMMONDEFS_ONLY +#ifndef LZ4_SRC_INCLUDED +#include "lz4.c" /* LZ4_count, constants, mem */ #endif -#define A64(x) (((U64_S *)(x))->v) -#define A32(x) (((U32_S *)(x))->v) -#define A16(x) (((U16_S *)(x))->v) - - -/************************************** - Constants -**************************************/ -#define MINMATCH 4 - -#define DICTIONARY_LOGSIZE 16 -#define MAXD (1< (b) ? (a) : (b) ) +#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) +#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ +#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */ +/* Make fields passed to, and updated by LZ4HC_encodeSequence explicit */ +#define UPDATABLE(ip, op, anchor) &ip, &op, &anchor -/************************************** - Architecture-specific macros -**************************************/ -#if LZ4_ARCH64 /* 64-bit */ -# define STEPSIZE 8 -# define LZ4_COPYSTEP(s,d) A64(d) = A64(s); d+=8; s+=8; -# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d) -# define AARCH A64 -# define HTYPE U32 -# define INITBASE(b,s) const BYTE* const b = s -#else /* 32-bit */ -# define STEPSIZE 4 -# define LZ4_COPYSTEP(s,d) A32(d) = A32(s); d+=4; s+=4; -# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d); LZ4_COPYSTEP(s,d); -# define AARCH A32 -# define HTYPE U32 -# define INITBASE(b,s) const BYTE* const b = s -#endif - -#if defined(LZ4_BIG_ENDIAN) -# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; } -# define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; } -#else /* Little Endian */ -# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); } -# define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; } -#endif +static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } /************************************** - Local Types +* HC Compression **************************************/ -typedef struct -{ - const BYTE* inputBuffer; - const BYTE* base; - const BYTE* end; - HTYPE hashTable[HASHTABLESIZE]; - U16 chainTable[MAXD]; - const BYTE* nextToUpdate; -} LZ4HC_Data_Structure; - - -/************************************** - Macros -**************************************/ -#define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d> ((MINMATCH*8)-HASH_LOG)) -#define HASH_VALUE(p) HASH_FUNCTION(A32(p)) -#define HASH_POINTER(p) (HashTable[HASH_VALUE(p)] + base) -#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK] -#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p)) - - -/************************************** - Private functions -**************************************/ -#if LZ4_ARCH64 - -FORCE_INLINE int LZ4_NbCommonBytes (register U64 val) -{ -#if defined(LZ4_BIG_ENDIAN) -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll(val) >> 3); -# else - int r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif -#else -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward64( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll(val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -val) * 0x0218A392CDABBD3F)) >> 58]; -# endif -#endif -} - -#else - -FORCE_INLINE int LZ4_NbCommonBytes (register U32 val) -{ -#if defined(LZ4_BIG_ENDIAN) -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanReverse( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz(val) >> 3); -# else - int r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif -#else -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz(val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif -#endif -} - -#endif - - -int LZ4_sizeofStreamStateHC() -{ - return sizeof(LZ4HC_Data_Structure); -} - -FORCE_INLINE void LZ4_initHC (LZ4HC_Data_Structure* hc4, const BYTE* base) +static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); - hc4->nextToUpdate = base + 1; - hc4->base = base; - hc4->inputBuffer = base; - hc4->end = base; } -int LZ4_resetStreamStateHC(void* state, const char* inputBuffer) +static void LZ4HC_init_internal (LZ4HC_CCtx_internal* hc4, const BYTE* start) { - if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ - LZ4_initHC((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); - return 0; + uptrval startingOffset = (uptrval)(hc4->end - hc4->base); + if (startingOffset > 1 GB) { + LZ4HC_clearTables(hc4); + startingOffset = 0; + } + startingOffset += 64 KB; + hc4->nextToUpdate = (U32) startingOffset; + hc4->base = start - startingOffset; + hc4->end = start; + hc4->dictBase = start - startingOffset; + hc4->dictLimit = (U32) startingOffset; + hc4->lowLimit = (U32) startingOffset; } -void* LZ4_createHC (const char* inputBuffer) +/* Update chains up to ip (excluded) */ +LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) { - void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure)); - LZ4_initHC ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer); - return hc4; -} + U16* const chainTable = hc4->chainTable; + U32* const hashTable = hc4->hashTable; + const BYTE* const base = hc4->base; + U32 const target = (U32)(ip - base); + U32 idx = hc4->nextToUpdate; + + while (idx < target) { + U32 const h = LZ4HC_hashPtr(base+idx); + size_t delta = idx - hashTable[h]; + if (delta>LZ4_DISTANCE_MAX) delta = LZ4_DISTANCE_MAX; + DELTANEXTU16(chainTable, idx) = (U16)delta; + hashTable[h] = idx; + idx++; + } + hc4->nextToUpdate = target; +} -int LZ4_freeHC (void* LZ4HC_Data) +/** LZ4HC_countBack() : + * @return : negative value, nb of common bytes before ip/match */ +LZ4_FORCE_INLINE +int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match, + const BYTE* const iMin, const BYTE* const mMin) { - FREEMEM(LZ4HC_Data); - return (0); + int back = 0; + int const min = (int)MAX(iMin - ip, mMin - match); + assert(min <= 0); + assert(ip >= iMin); assert((size_t)(ip-iMin) < (1U<<31)); + assert(match >= mMin); assert((size_t)(match - mMin) < (1U<<31)); + while ( (back > min) + && (ip[back-1] == match[back-1]) ) + back--; + return back; } - -/* Update chains up to ip (excluded) */ -FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) -{ - U16* chainTable = hc4->chainTable; - HTYPE* HashTable = hc4->hashTable; - INITBASE(base,hc4->base); - - while(hc4->nextToUpdate < ip) - { - const BYTE* const p = hc4->nextToUpdate; - size_t delta = (p) - HASH_POINTER(p); - if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; - DELTANEXT(p) = (U16)delta; - HashTable[HASH_VALUE(p)] = (HTYPE)((p) - base); - hc4->nextToUpdate++; - } -} +#if defined(_MSC_VER) +# define LZ4HC_rotl32(x,r) _rotl(x,r) +#else +# define LZ4HC_rotl32(x,r) ((x << r) | (x >> (32 - r))) +#endif -char* LZ4_slideInputBufferHC(void* LZ4HC_Data) +static U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern) { - LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; - size_t distance = (hc4->end - 64 KB) - hc4->inputBuffer; - - if (hc4->end <= hc4->inputBuffer + 64 KB) return (char*)(hc4->end); /* no update : less than 64KB within buffer */ - - distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */ - LZ4HC_Insert(hc4, hc4->end - MINMATCH); - memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB); - hc4->nextToUpdate -= distance; - hc4->base -= distance; - if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */ - { - int i; - hc4->base += 1 GB; - for (i=0; ihashTable[i] -= 1 GB; - } - hc4->end -= distance; - return (char*)(hc4->end); + size_t const bitsToRotate = (rotate & (sizeof(pattern) - 1)) << 3; + if (bitsToRotate == 0) + return pattern; + return LZ4HC_rotl32(pattern, (int)bitsToRotate); } - -FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit) +/* LZ4HC_countPattern() : + * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */ +static unsigned +LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32) { - const BYTE* p1t = p1; - - while (p1tchainTable; - HTYPE* const HashTable = hc4->hashTable; - const BYTE* ref; - INITBASE(base,hc4->base); - int nbAttempts=maxNbAttempts; - size_t repl=0, ml=0; - U16 delta=0; /* useless assignment, to remove an uninitialization warning */ - - /* HC4 match finder */ - LZ4HC_Insert(hc4, ip); - ref = HASH_POINTER(ip); - -#define REPEAT_OPTIMIZATION -#ifdef REPEAT_OPTIMIZATION - /* Detect repetitive sequences of length <= 4 */ - if ((U32)(ip-ref) <= 4) /* potential repetition */ - { - if (A32(ref) == A32(ip)) /* confirmed */ - { - delta = (U16)(ip-ref); - repl = ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH; - *matchpos = ref; + if (LZ4_isLittleEndian()) { + reg_t patternByte = pattern; + while ((ip>= 8; } - ref = GETNEXT(ref); - } -#endif - - while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts)) - { - nbAttempts--; - if (*(ref+ml) == *(ip+ml)) - if (A32(ref) == A32(ip)) - { - size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH; - if (mlt > ml) { ml = mlt; *matchpos = ref; } + } else { /* big endian */ + U32 bitOffset = (sizeof(pattern)*8) - 8; + while (ip < iEnd) { + BYTE const byte = (BYTE)(pattern >> bitOffset); + if (*ip != byte) break; + ip ++; bitOffset -= 8; } - ref = GETNEXT(ref); } -#ifdef REPEAT_OPTIMIZATION - /* Complete table */ - if (repl) - { - const BYTE* ptr = ip; - const BYTE* end; - - end = ip + repl - (MINMATCH-1); - while(ptr < end-delta) - { - DELTANEXT(ptr) = delta; /* Pre-Load */ - ptr++; - } - do - { - DELTANEXT(ptr) = delta; - HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); /* Head of chain */ - ptr++; - } while(ptr < end); - hc4->nextToUpdate = end; - } -#endif + return (unsigned)(ip - iStart); +} + +/* LZ4HC_reverseCountPattern() : + * pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) + * read using natural platform endianess */ +static unsigned +LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern) +{ + const BYTE* const iStart = ip; - return (int)ml; + while (likely(ip >= iLow+4)) { + if (LZ4_read32(ip-4) != pattern) break; + ip -= 4; + } + { const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianess */ + while (likely(ip>iLow)) { + if (ip[-1] != *bytePtr) break; + ip--; bytePtr--; + } } + return (unsigned)(iStart - ip); } +/* LZ4HC_protectDictEnd() : + * Checks if the match is in the last 3 bytes of the dictionary, so reading the + * 4 byte MINMATCH would overflow. + * @returns true if the match index is okay. + */ +static int LZ4HC_protectDictEnd(U32 const dictLimit, U32 const matchIndex) +{ + return ((U32)((dictLimit - 1) - matchIndex) >= 3); +} -FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos, const int maxNbAttempts) +typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e; +typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e; + +LZ4_FORCE_INLINE int +LZ4HC_InsertAndGetWiderMatch ( + LZ4HC_CCtx_internal* hc4, + const BYTE* const ip, + const BYTE* const iLowLimit, + const BYTE* const iHighLimit, + int longest, + const BYTE** matchpos, + const BYTE** startpos, + const int maxNbAttempts, + const int patternAnalysis, + const int chainSwap, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) { - U16* const chainTable = hc4->chainTable; - HTYPE* const HashTable = hc4->hashTable; - INITBASE(base,hc4->base); - const BYTE* ref; + U16* const chainTable = hc4->chainTable; + U32* const HashTable = hc4->hashTable; + const LZ4HC_CCtx_internal * const dictCtx = hc4->dictCtx; + const BYTE* const base = hc4->base; + const U32 dictLimit = hc4->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const U32 ipIndex = (U32)(ip - base); + const U32 lowestMatchIndex = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX; + const BYTE* const dictBase = hc4->dictBase; + int const lookBackLength = (int)(ip-iLowLimit); int nbAttempts = maxNbAttempts; - int delta = (int)(ip-startLimit); + U32 matchChainPos = 0; + U32 const pattern = LZ4_read32(ip); + U32 matchIndex; + repeat_state_e repeat = rep_untested; + size_t srcPatternLength = 0; + DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch"); /* First Match */ LZ4HC_Insert(hc4, ip); - ref = HASH_POINTER(ip); + matchIndex = HashTable[LZ4HC_hashPtr(ip)]; + DEBUGLOG(7, "First match at index %u / %u (lowestMatchIndex)", + matchIndex, lowestMatchIndex); - while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts)) - { + while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) { + int matchLength=0; nbAttempts--; - if (*(startLimit + longest) == *(ref - delta + longest)) - if (A32(ref) == A32(ip)) - { -#if 1 - const BYTE* reft = ref+MINMATCH; - const BYTE* ipt = ip+MINMATCH; - const BYTE* startt = ip; - - while (iptstartLimit) && (reft > hc4->inputBuffer) && (startt[-1] == reft[-1])) {startt--; reft--;} - - if ((ipt-startt) > longest) - { - longest = (int)(ipt-startt); - *matchpos = reft; - *startpos = startt; - } - } - ref = GETNEXT(ref); - } + assert(matchIndex < ipIndex); + if (favorDecSpeed && (ipIndex - matchIndex < 8)) { + /* do nothing */ + } else if (matchIndex >= dictLimit) { /* within current Prefix */ + const BYTE* const matchPtr = base + matchIndex; + assert(matchPtr >= lowPrefixPtr); + assert(matchPtr < ip); + assert(longest >= 1); + if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) { + if (LZ4_read32(matchPtr) == pattern) { + int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0; + matchLength = MINMATCH + (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); + matchLength -= back; + if (matchLength > longest) { + longest = matchLength; + *matchpos = matchPtr + back; + *startpos = ip + back; + } } } + } else { /* lowestMatchIndex <= matchIndex < dictLimit */ + const BYTE* const matchPtr = dictBase + matchIndex; + if (LZ4_read32(matchPtr) == pattern) { + const BYTE* const dictStart = dictBase + hc4->lowLimit; + int back = 0; + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + matchLength = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + if ((ip+matchLength == vLimit) && (vLimit < iHighLimit)) + matchLength += LZ4_count(ip+matchLength, lowPrefixPtr, iHighLimit); + back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0; + matchLength -= back; + if (matchLength > longest) { + longest = matchLength; + *matchpos = base + matchIndex + back; /* virtual pos, relative to ip, to retrieve offset */ + *startpos = ip + back; + } } } + + if (chainSwap && matchLength==longest) { /* better match => select a better chain */ + assert(lookBackLength==0); /* search forward only */ + if (matchIndex + (U32)longest <= ipIndex) { + int const kTrigger = 4; + U32 distanceToNextMatch = 1; + int const end = longest - MINMATCH + 1; + int step = 1; + int accel = 1 << kTrigger; + int pos; + for (pos = 0; pos < end; pos += step) { + U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos); + step = (accel++ >> kTrigger); + if (candidateDist > distanceToNextMatch) { + distanceToNextMatch = candidateDist; + matchChainPos = (U32)pos; + accel = 1 << kTrigger; + } + } + if (distanceToNextMatch > 1) { + if (distanceToNextMatch > matchIndex) break; /* avoid overflow */ + matchIndex -= distanceToNextMatch; + continue; + } } } + + { U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex); + if (patternAnalysis && distNextMatch==1 && matchChainPos==0) { + U32 const matchCandidateIdx = matchIndex-1; + /* may be a repeated pattern */ + if (repeat == rep_untested) { + if ( ((pattern & 0xFFFF) == (pattern >> 16)) + & ((pattern & 0xFF) == (pattern >> 24)) ) { + repeat = rep_confirmed; + srcPatternLength = LZ4HC_countPattern(ip+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern); + } else { + repeat = rep_not; + } } + if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex) + && LZ4HC_protectDictEnd(dictLimit, matchCandidateIdx) ) { + const int extDict = matchCandidateIdx < dictLimit; + const BYTE* const matchPtr = (extDict ? dictBase : base) + matchCandidateIdx; + if (LZ4_read32(matchPtr) == pattern) { /* good candidate */ + const BYTE* const dictStart = dictBase + hc4->lowLimit; + const BYTE* const iLimit = extDict ? dictBase + dictLimit : iHighLimit; + size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern); + if (extDict && matchPtr + forwardPatternLength == iLimit) { + U32 const rotatedPattern = LZ4HC_rotatePattern(forwardPatternLength, pattern); + forwardPatternLength += LZ4HC_countPattern(lowPrefixPtr, iHighLimit, rotatedPattern); + } + { const BYTE* const lowestMatchPtr = extDict ? dictStart : lowPrefixPtr; + size_t backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern); + size_t currentSegmentLength; + if (!extDict && matchPtr - backLength == lowPrefixPtr && hc4->lowLimit < dictLimit) { + U32 const rotatedPattern = LZ4HC_rotatePattern((U32)(-(int)backLength), pattern); + backLength += LZ4HC_reverseCountPattern(dictBase + dictLimit, dictStart, rotatedPattern); + } + /* Limit backLength not go further than lowestMatchIndex */ + backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLength, lowestMatchIndex); + assert(matchCandidateIdx - backLength >= lowestMatchIndex); + currentSegmentLength = backLength + forwardPatternLength; + /* Adjust to end of pattern if the source pattern fits, otherwise the beginning of the pattern */ + if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ + && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */ + U32 const newMatchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ + if (LZ4HC_protectDictEnd(dictLimit, newMatchIndex)) + matchIndex = newMatchIndex; + else { + /* Can only happen if started in the prefix */ + assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict); + matchIndex = dictLimit; + } + } else { + U32 const newMatchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */ + if (!LZ4HC_protectDictEnd(dictLimit, newMatchIndex)) { + assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict); + matchIndex = dictLimit; + } else { + matchIndex = newMatchIndex; + if (lookBackLength==0) { /* no back possible */ + size_t const maxML = MIN(currentSegmentLength, srcPatternLength); + if ((size_t)longest < maxML) { + assert(base + matchIndex < ip); + if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break; + assert(maxML < 2 GB); + longest = (int)maxML; + *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ + *startpos = ip; + } + { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex); + if (distToNextPattern > matchIndex) break; /* avoid overflow */ + matchIndex -= distToNextPattern; + } } } } } + continue; + } } + } } /* PA optimization */ + + /* follow current chain */ + matchIndex -= DELTANEXTU16(chainTable, matchIndex + matchChainPos); + + } /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */ + + if ( dict == usingDictCtxHc + && nbAttempts + && ipIndex - lowestMatchIndex < LZ4_DISTANCE_MAX) { + size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->base); + U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; + assert(dictEndOffset <= 1 GB); + matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset; + while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) { + const BYTE* const matchPtr = dictCtx->base + dictMatchIndex; + + if (LZ4_read32(matchPtr) == pattern) { + int mlt; + int back = 0; + const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0; + mlt -= back; + if (mlt > longest) { + longest = mlt; + *matchpos = base + matchIndex + back; + *startpos = ip + back; + } } + + { U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex); + dictMatchIndex -= nextOffset; + matchIndex -= nextOffset; + } } } return longest; } +LZ4_FORCE_INLINE +int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */ + const BYTE* const ip, const BYTE* const iLimit, + const BYTE** matchpos, + const int maxNbAttempts, + const int patternAnalysis, + const dictCtx_directive dict) +{ + const BYTE* uselessPtr = ip; + /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), + * but this won't be the case here, as we define iLowLimit==ip, + * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ + return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, 0 /*chainSwap*/, dict, favorCompressionRatio); +} -typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; - -FORCE_INLINE int LZ4HC_encodeSequence ( - const BYTE** ip, - BYTE** op, - const BYTE** anchor, - int matchLength, - const BYTE* ref, - limitedOutput_directive limitedOutputBuffer, - BYTE* oend) +/* LZ4HC_encodeSequence() : + * @return : 0 if ok, + * 1 if buffer issue detected */ +LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( + const BYTE** ip, + BYTE** op, + const BYTE** anchor, + int matchLength, + const BYTE* const match, + limitedOutput_directive limit, + BYTE* oend) { - int length; - BYTE* token; + size_t length; + BYTE* const token = (*op)++; + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 6) + static const BYTE* start = NULL; + static U32 totalCost = 0; + U32 const pos = (start==NULL) ? 0 : (U32)(*anchor - start); + U32 const ll = (U32)(*ip - *anchor); + U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0; + U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0; + U32 const cost = 1 + llAdd + ll + 2 + mlAdd; + if (start==NULL) start = *anchor; /* only works for single segment */ + /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */ + DEBUGLOG(6, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u", + pos, + (U32)(*ip - *anchor), matchLength, (U32)(*ip-match), + cost, totalCost); + totalCost += cost; +#endif /* Encode Literal length */ - length = (int)(*ip - *anchor); - token = (*op)++; - if ((limitedOutputBuffer) && ((*op + length + (2 + 1 + LASTLITERALS) + (length>>8)) > oend)) return 1; /* Check output limit */ - if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK< 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } - else *token = (BYTE)(length< oend)) return 1; /* Check output limit */ + if (length >= RUN_MASK) { + size_t len = length - RUN_MASK; + *token = (RUN_MASK << ML_BITS); + for(; len >= 255 ; len -= 255) *(*op)++ = 255; + *(*op)++ = (BYTE)len; + } else { + *token = (BYTE)(length << ML_BITS); + } /* Copy Literals */ - LZ4_BLINDCOPY(*anchor, *op, length); + LZ4_wildCopy8(*op, *anchor, (*op) + length); + *op += length; /* Encode Offset */ - LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-ref)); + assert( (*ip - match) <= LZ4_DISTANCE_MAX ); /* note : consider providing offset as a value, rather than as a pointer difference */ + LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2; /* Encode MatchLength */ - length = (int)(matchLength-MINMATCH); - if ((limitedOutputBuffer) && (*op + (1 + LASTLITERALS) + (length>>8) > oend)) return 1; /* Check output limit */ - if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; } - else *token += (BYTE)(length); + assert(matchLength >= MINMATCH); + length = (size_t)matchLength - MINMATCH; + if ((limit) && (*op + (length / 255) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ + if (length >= ML_MASK) { + *token += ML_MASK; + length -= ML_MASK; + for(; length >= 510 ; length -= 510) { *(*op)++ = 255; *(*op)++ = 255; } + if (length >= 255) { length -= 255; *(*op)++ = 255; } + *(*op)++ = (BYTE)length; + } else { + *token += (BYTE)(length); + } /* Prepare next loop */ *ip += matchLength; @@ -607,80 +529,74 @@ FORCE_INLINE int LZ4HC_encodeSequence ( return 0; } - -#define MAX_COMPRESSION_LEVEL 16 -static int LZ4HC_compress_generic ( - void* ctxvoid, - const char* source, - char* dest, - int inputSize, - int maxOutputSize, - int compressionLevel, - limitedOutput_directive limit - ) +LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( + LZ4HC_CCtx_internal* const ctx, + const char* const source, + char* const dest, + int* srcSizePtr, + int const maxOutputSize, + unsigned maxNbAttempts, + const limitedOutput_directive limit, + const dictCtx_directive dict + ) { - LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid; + const int inputSize = *srcSizePtr; + const int patternAnalysis = (maxNbAttempts > 128); /* levels 9+ */ + const BYTE* ip = (const BYTE*) source; const BYTE* anchor = ip; const BYTE* const iend = ip + inputSize; const BYTE* const mflimit = iend - MFLIMIT; const BYTE* const matchlimit = (iend - LASTLITERALS); + BYTE* optr = (BYTE*) dest; BYTE* op = (BYTE*) dest; - BYTE* const oend = op + maxOutputSize; - - const int maxNbAttempts = compressionLevel > MAX_COMPRESSION_LEVEL ? 1 << MAX_COMPRESSION_LEVEL : compressionLevel ? 1<<(compressionLevel-1) : 1<end) return 0; - ctx->end += inputSize; - - ip++; + /* init */ + *srcSizePtr = 0; + if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ /* Main Loop */ - while (ip < mflimit) - { - ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts); - if (!ml) { ip++; continue; } + while (ip <= mflimit) { + ml = LZ4HC_InsertAndFindBestMatch(ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis, dict); + if (ml encode ML1 */ + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; continue; } - if (start0 < ip) - { - if (start2 < ip + ml0) /* empirical */ - { - ip = start0; - ref = ref0; - ml = ml0; - } - } + if (start0 < ip) { /* first match was skipped at least once */ + if (start2 < ip + ml0) { /* squeezing ML1 between ML0(original ML1) and ML2 */ + ip = start0; ref = ref0; ml = ml0; /* restore initial ML1 */ + } } /* Here, start0==ip */ - if ((start2 - ip) < 3) /* First Match too small : removed */ - { + if ((start2 - ip) < 3) { /* First Match too small : removed */ ml = ml2; ip = start2; ref =ref2; @@ -688,20 +604,16 @@ static int LZ4HC_compress_generic ( } _Search3: - /* - * Currently we have : - * ml2 > ml1, and - * ip1+3 <= ip2 (usually < ip1+ml1) - */ - if ((start2 - ip) < OPTIMAL_ML) - { + /* At this stage, we have : + * ml2 > ml1, and + * ip1+3 <= ip2 (usually < ip1+ml1) */ + if ((start2 - ip) < OPTIMAL_ML) { int correction; int new_ml = ml; if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; correction = new_ml - (int)(start2 - ip); - if (correction > 0) - { + if (correction > 0) { start2 += correction; ref2 += correction; ml2 -= correction; @@ -709,40 +621,42 @@ static int LZ4HC_compress_generic ( } /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ - if (start2 + ml2 < mflimit) - ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts); - else ml3 = ml2; + if (start2 + ml2 <= mflimit) { + ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, + start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, + maxNbAttempts, patternAnalysis, 0, dict, favorCompressionRatio); + } else { + ml3 = ml2; + } - if (ml3 == ml2) /* No better match : 2 sequences to encode */ - { + if (ml3 == ml2) { /* No better match => encode ML1 and ML2 */ /* ip & ref are known; Now for ml */ if (start2 < ip+ml) ml = (int)(start2 - ip); /* Now, encode 2 sequences */ - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; ip = start2; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0; + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml2, ref2, limit, oend)) goto _dest_overflow; continue; } - if (start3 < ip+ml+3) /* Not enough space for match 2 : remove it */ - { - if (start3 >= (ip+ml)) /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ - { - if (start2 < ip+ml) - { + if (start3 < ip+ml+3) { /* Not enough space for match 2 : remove it */ + if (start3 >= (ip+ml)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ + if (start2 < ip+ml) { int correction = (int)(ip+ml - start2); start2 += correction; ref2 += correction; ml2 -= correction; - if (ml2 < MINMATCH) - { + if (ml2 < MINMATCH) { start2 = start3; ref2 = ref3; ml2 = ml3; } } - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; ip = start3; ref = ref3; ml = ml3; @@ -760,138 +674,865 @@ static int LZ4HC_compress_generic ( } /* - * OK, now we have 3 ascending matches; let's write at least the first one - * ip & ref are known; Now for ml - */ - if (start2 < ip+ml) - { - if ((start2 - ip) < (int)ML_MASK) - { + * OK, now we have 3 ascending matches; + * let's write the first one ML1. + * ip & ref are known; Now decide ml. + */ + if (start2 < ip+ml) { + if ((start2 - ip) < OPTIMAL_ML) { int correction; if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH; correction = ml - (int)(start2 - ip); - if (correction > 0) - { + if (correction > 0) { start2 += correction; ref2 += correction; ml2 -= correction; } - } - else - { + } else { ml = (int)(start2 - ip); } } - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; - ip = start2; - ref = ref2; - ml = ml2; + /* ML2 becomes ML1 */ + ip = start2; ref = ref2; ml = ml2; - start2 = start3; - ref2 = ref3; - ml2 = ml3; + /* ML3 becomes ML2 */ + start2 = start3; ref2 = ref3; ml2 = ml3; + /* let's find a new ML3 */ goto _Search3; - } +_last_literals: /* Encode Last Literals */ - { - int lastRun = (int)(iend - anchor); - if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ - if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } - else *op++ = (BYTE)(lastRun< oend)) { + if (limit == limitedOutput) return 0; /* Check output limit */ + /* adapt lastRunSize to fill 'dest' */ + lastRunSize = (size_t)(oend - op) - 1; + litLength = (lastRunSize + 255 - RUN_MASK) / 255; + lastRunSize -= litLength; + } + ip = anchor + lastRunSize; + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + memcpy(op, anchor, lastRunSize); + op += lastRunSize; } /* End */ + *srcSizePtr = (int) (((const char*)ip) - source); return (int) (((char*)op)-dest); + +_dest_overflow: + if (limit == fillOutput) { + op = optr; /* restore correct out pointer */ + goto _last_literals; + } + return 0; } -int LZ4_compressHC2(const char* source, char* dest, int inputSize, int compressionLevel) +static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx, + const char* const source, char* dst, + int* srcSizePtr, int dstCapacity, + int const nbSearches, size_t sufficient_len, + const limitedOutput_directive limit, int const fullUpdate, + const dictCtx_directive dict, + HCfavor_e favorDecSpeed); + + +LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + const limitedOutput_directive limit, + const dictCtx_directive dict + ) +{ + typedef enum { lz4hc, lz4opt } lz4hc_strat_e; + typedef struct { + lz4hc_strat_e strat; + U32 nbSearches; + U32 targetLength; + } cParams_t; + static const cParams_t clTable[LZ4HC_CLEVEL_MAX+1] = { + { lz4hc, 2, 16 }, /* 0, unused */ + { lz4hc, 2, 16 }, /* 1, unused */ + { lz4hc, 2, 16 }, /* 2, unused */ + { lz4hc, 4, 16 }, /* 3 */ + { lz4hc, 8, 16 }, /* 4 */ + { lz4hc, 16, 16 }, /* 5 */ + { lz4hc, 32, 16 }, /* 6 */ + { lz4hc, 64, 16 }, /* 7 */ + { lz4hc, 128, 16 }, /* 8 */ + { lz4hc, 256, 16 }, /* 9 */ + { lz4opt, 96, 64 }, /*10==LZ4HC_CLEVEL_OPT_MIN*/ + { lz4opt, 512,128 }, /*11 */ + { lz4opt,16384,LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ + }; + + DEBUGLOG(4, "LZ4HC_compress_generic(ctx=%p, src=%p, srcSize=%d)", ctx, src, *srcSizePtr); + + if (limit == fillOutput && dstCapacity < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */ + + ctx->end += *srcSizePtr; + if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe something to review */ + cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel); + { cParams_t const cParam = clTable[cLevel]; + HCfavor_e const favor = ctx->favorDecSpeed ? favorDecompressionSpeed : favorCompressionRatio; + int result; + + if (cParam.strat == lz4hc) { + result = LZ4HC_compress_hashChain(ctx, + src, dst, srcSizePtr, dstCapacity, + cParam.nbSearches, limit, dict); + } else { + assert(cParam.strat == lz4opt); + result = LZ4HC_compress_optimal(ctx, + src, dst, srcSizePtr, dstCapacity, + (int)cParam.nbSearches, cParam.targetLength, limit, + cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */ + dict, favor); + } + if (result <= 0) ctx->dirty = 1; + return result; + } +} + +static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock); + +static int +LZ4HC_compress_generic_noDictCtx ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) { - void* ctx = LZ4_createHC(source); - int result; - if (ctx==NULL) return 0; + assert(ctx->dictCtx == NULL); + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, noDictCtx); +} - result = LZ4HC_compress_generic (ctx, source, dest, inputSize, 0, compressionLevel, noLimit); +static int +LZ4HC_compress_generic_dictCtx ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + const size_t position = (size_t)(ctx->end - ctx->base) - ctx->lowLimit; + assert(ctx->dictCtx != NULL); + if (position >= 64 KB) { + ctx->dictCtx = NULL; + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else if (position == 0 && *srcSizePtr > 4 KB) { + memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal)); + LZ4HC_setExternalDict(ctx, (const BYTE *)src); + ctx->compressionLevel = (short)cLevel; + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else { + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtxHc); + } +} - LZ4_freeHC(ctx); - return result; +static int +LZ4HC_compress_generic ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + if (ctx->dictCtx == NULL) { + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else { + return LZ4HC_compress_generic_dictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } } -int LZ4_compressHC(const char* source, char* dest, int inputSize) { return LZ4_compressHC2(source, dest, inputSize, 0); } -int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) +int LZ4_sizeofStateHC(void) { return (int)sizeof(LZ4_streamHC_t); } + +#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : + * it reports an aligment of 8-bytes, + * while actually aligning LZ4_streamHC_t on 4 bytes. */ +static size_t LZ4_streamHC_t_alignment(void) { - void* ctx = LZ4_createHC(source); - int result; - if (ctx==NULL) return 0; + struct { char c; LZ4_streamHC_t t; } t_a; + return sizeof(t_a) - sizeof(t_a.t); +} +#endif + +/* state is presumed correctly initialized, + * in which case its size and alignment have already been validate */ +int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +{ + LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; +#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : + * it reports an aligment of 8-bytes, + * while actually aligning LZ4_streamHC_t on 4 bytes. */ + assert(((size_t)state & (LZ4_streamHC_t_alignment() - 1)) == 0); /* check alignment */ +#endif + if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ + LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel); + LZ4HC_init_internal (ctx, (const BYTE*)src); + if (dstCapacity < LZ4_compressBound(srcSize)) + return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput); + else + return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, notLimited); +} - result = LZ4HC_compress_generic (ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); +int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +{ + LZ4_streamHC_t* const ctx = LZ4_initStreamHC(state, sizeof(*ctx)); + if (ctx==NULL) return 0; /* init failure */ + return LZ4_compress_HC_extStateHC_fastReset(state, src, dst, srcSize, dstCapacity, compressionLevel); +} - LZ4_freeHC(ctx); - return result; +int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +{ +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); +#else + LZ4_streamHC_t state; + LZ4_streamHC_t* const statePtr = &state; +#endif + int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + FREEMEM(statePtr); +#endif + return cSize; } -int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +/* state is presumed sized correctly (>= sizeof(LZ4_streamHC_t)) */ +int LZ4_compress_HC_destSize(void* state, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel) { - return LZ4_compressHC2_limitedOutput(source, dest, inputSize, maxOutputSize, 0); + LZ4_streamHC_t* const ctx = LZ4_initStreamHC(state, sizeof(*ctx)); + if (ctx==NULL) return 0; /* init failure */ + LZ4HC_init_internal(&ctx->internal_donotuse, (const BYTE*) source); + LZ4_setCompressionLevel(ctx, cLevel); + return LZ4HC_compress_generic(&ctx->internal_donotuse, source, dest, sourceSizePtr, targetDestSize, cLevel, fillOutput); } -/***************************** - Using external allocation -*****************************/ -int LZ4_sizeofStateHC() { return sizeof(LZ4HC_Data_Structure); } +/************************************** +* Streaming Functions +**************************************/ +/* allocation */ +LZ4_streamHC_t* LZ4_createStreamHC(void) +{ + LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); + if (LZ4_streamHCPtr==NULL) return NULL; + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); /* full initialization, malloc'ed buffer can be full of garbage */ + return LZ4_streamHCPtr; +} -int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel) +int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { - if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ - LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source); - return LZ4HC_compress_generic (state, source, dest, inputSize, 0, compressionLevel, noLimit); + DEBUGLOG(4, "LZ4_freeStreamHC(%p)", LZ4_streamHCPtr); + if (!LZ4_streamHCPtr) return 0; /* support free on NULL */ + FREEMEM(LZ4_streamHCPtr); + return 0; } -int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize) -{ return LZ4_compressHC2_withStateHC (state, source, dest, inputSize, 0); } +LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size) +{ + LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)buffer; + if (buffer == NULL) return NULL; + if (size < sizeof(LZ4_streamHC_t)) return NULL; +#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : + * it reports an aligment of 8-bytes, + * while actually aligning LZ4_streamHC_t on 4 bytes. */ + if (((size_t)buffer) & (LZ4_streamHC_t_alignment() - 1)) return NULL; /* alignment check */ +#endif + /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ + LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= LZ4_STREAMHCSIZE); + DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", LZ4_streamHCPtr, (unsigned)size); + /* end-base will trigger a clearTable on starting compression */ + LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1; + LZ4_streamHCPtr->internal_donotuse.base = NULL; + LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; + LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = 0; + LZ4_streamHCPtr->internal_donotuse.dirty = 0; + LZ4_setCompressionLevel(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT); + return LZ4_streamHCPtr; +} -int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) +/* just a stub */ +void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { - if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ - LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source); - return LZ4HC_compress_generic (state, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); } -int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) -{ return LZ4_compressHC2_limitedOutput_withStateHC (state, source, dest, inputSize, maxOutputSize, 0); } +void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel); + if (LZ4_streamHCPtr->internal_donotuse.dirty) { + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + } else { + /* preserve end - base : can trigger clearTable's threshold */ + LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base; + LZ4_streamHCPtr->internal_donotuse.base = NULL; + LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; + } + LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); +} +void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + DEBUGLOG(5, "LZ4_setCompressionLevel(%p, %d)", LZ4_streamHCPtr, compressionLevel); + if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; + if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX; + LZ4_streamHCPtr->internal_donotuse.compressionLevel = (short)compressionLevel; +} -/**************************** - Stream functions -****************************/ +void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor) +{ + LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = (favor!=0); +} -int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize) +/* LZ4_loadDictHC() : + * LZ4_streamHCPtr is presumed properly initialized */ +int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, + const char* dictionary, int dictSize) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit); + LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + DEBUGLOG(4, "LZ4_loadDictHC(%p, %p, %d)", LZ4_streamHCPtr, dictionary, dictSize); + assert(LZ4_streamHCPtr != NULL); + if (dictSize > 64 KB) { + dictionary += (size_t)dictSize - 64 KB; + dictSize = 64 KB; + } + /* need a full initialization, there are bad side-effects when using resetFast() */ + { int const cLevel = ctxPtr->compressionLevel; + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + LZ4_setCompressionLevel(LZ4_streamHCPtr, cLevel); + } + LZ4HC_init_internal (ctxPtr, (const BYTE*)dictionary); + ctxPtr->end = (const BYTE*)dictionary + dictSize; + if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); + return dictSize; +} + +void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream) { + working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL; } -int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) +/* compression */ + +static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit); + DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock); + if (ctxPtr->end >= ctxPtr->base + ctxPtr->dictLimit + 4) + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); + ctxPtr->dictBase = ctxPtr->base; + ctxPtr->base = newBlock - ctxPtr->dictLimit; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ + + /* cannot reference an extDict and a dictCtx at the same time */ + ctxPtr->dictCtx = NULL; } -int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize) +static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, + const char* src, char* dst, + int* srcSizePtr, int dstCapacity, + limitedOutput_directive limit) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput); + LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + DEBUGLOG(4, "LZ4_compressHC_continue_generic(ctx=%p, src=%p, srcSize=%d)", + LZ4_streamHCPtr, src, *srcSizePtr); + assert(ctxPtr != NULL); + /* auto-init if forgotten */ + if (ctxPtr->base == NULL) LZ4HC_init_internal (ctxPtr, (const BYTE*) src); + + /* Check overflow */ + if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { + size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; + if (dictSize > 64 KB) dictSize = 64 KB; + LZ4_loadDictHC(LZ4_streamHCPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); + } + + /* Check if blocks follow each other */ + if ((const BYTE*)src != ctxPtr->end) + LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src); + + /* Check overlapping input/dictionary space */ + { const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr; + const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; + const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; + if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd)) { + if (sourceEnd > dictEnd) sourceEnd = dictEnd; + ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); + if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; + } + } + + return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit); } -int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) +int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int srcSize, int dstCapacity) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); + if (dstCapacity < LZ4_compressBound(srcSize)) + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, limitedOutput); + else + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, notLimited); } + +int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize) +{ + return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, fillOutput); +} + + + +/* dictionary saving */ + +int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) +{ + LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; + int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); + DEBUGLOG(4, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize); + if (dictSize > 64 KB) dictSize = 64 KB; + if (dictSize < 4) dictSize = 0; + if (dictSize > prefixSize) dictSize = prefixSize; + memmove(safeBuffer, streamPtr->end - dictSize, dictSize); + { U32 const endIndex = (U32)(streamPtr->end - streamPtr->base); + streamPtr->end = (const BYTE*)safeBuffer + dictSize; + streamPtr->base = streamPtr->end - endIndex; + streamPtr->dictLimit = endIndex - (U32)dictSize; + streamPtr->lowLimit = endIndex - (U32)dictSize; + if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; + } + return dictSize; +} + + +/*************************************************** +* Deprecated Functions +***************************************************/ + +/* These functions currently generate deprecation warnings */ + +/* Wrappers for deprecated compression functions */ +int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); } +int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); } +int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } +int LZ4_compressHC2_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, cLevel); } +int LZ4_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, LZ4_compressBound(srcSize), 0); } +int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); } +int LZ4_compressHC2_withStateHC (void* state, const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } +int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, maxDstSize, cLevel); } +int LZ4_compressHC_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, LZ4_compressBound(srcSize)); } +int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); } + + +/* Deprecated streaming functions */ +int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } + +/* state is presumed correctly sized, aka >= sizeof(LZ4_streamHC_t) + * @return : 0 on success, !=0 if error */ +int LZ4_resetStreamStateHC(void* state, char* inputBuffer) +{ + LZ4_streamHC_t* const hc4 = LZ4_initStreamHC(state, sizeof(*hc4)); + if (hc4 == NULL) return 1; /* init failed */ + LZ4HC_init_internal (&hc4->internal_donotuse, (const BYTE*)inputBuffer); + return 0; +} + +void* LZ4_createHC (const char* inputBuffer) +{ + LZ4_streamHC_t* const hc4 = LZ4_createStreamHC(); + if (hc4 == NULL) return NULL; /* not enough memory */ + LZ4HC_init_internal (&hc4->internal_donotuse, (const BYTE*)inputBuffer); + return hc4; +} + +int LZ4_freeHC (void* LZ4HC_Data) +{ + if (!LZ4HC_Data) return 0; /* support free on NULL */ + FREEMEM(LZ4HC_Data); + return 0; +} + +int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel) +{ + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, notLimited); +} + +int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int dstCapacity, int cLevel) +{ + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, dstCapacity, cLevel, limitedOutput); +} + +char* LZ4_slideInputBufferHC(void* LZ4HC_Data) +{ + LZ4_streamHC_t *ctx = (LZ4_streamHC_t*)LZ4HC_Data; + const BYTE *bufferStart = ctx->internal_donotuse.base + ctx->internal_donotuse.lowLimit; + LZ4_resetStreamHC_fast(ctx, ctx->internal_donotuse.compressionLevel); + /* avoid const char * -> char * conversion warning :( */ + return (char *)(uptrval)bufferStart; +} + + +/* ================================================ + * LZ4 Optimal parser (levels [LZ4HC_CLEVEL_OPT_MIN - LZ4HC_CLEVEL_MAX]) + * ===============================================*/ +typedef struct { + int price; + int off; + int mlen; + int litlen; +} LZ4HC_optimal_t; + +/* price in bytes */ +LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen) +{ + int price = litlen; + assert(litlen >= 0); + if (litlen >= (int)RUN_MASK) + price += 1 + ((litlen-(int)RUN_MASK) / 255); + return price; +} + + +/* requires mlen >= MINMATCH */ +LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen) +{ + int price = 1 + 2 ; /* token + 16-bit offset */ + assert(litlen >= 0); + assert(mlen >= MINMATCH); + + price += LZ4HC_literalsPrice(litlen); + + if (mlen >= (int)(ML_MASK+MINMATCH)) + price += 1 + ((mlen-(int)(ML_MASK+MINMATCH)) / 255); + + return price; +} + + +typedef struct { + int off; + int len; +} LZ4HC_match_t; + +LZ4_FORCE_INLINE LZ4HC_match_t +LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx, + const BYTE* ip, const BYTE* const iHighLimit, + int minLen, int nbSearches, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) +{ + LZ4HC_match_t match = { 0 , 0 }; + const BYTE* matchPtr = NULL; + /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), + * but this won't be the case here, as we define iLowLimit==ip, + * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ + int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /*patternAnalysis*/, 1 /*chainSwap*/, dict, favorDecSpeed); + if (matchLength <= minLen) return match; + if (favorDecSpeed) { + if ((matchLength>18) & (matchLength<=36)) matchLength=18; /* favor shortcut */ + } + match.len = matchLength; + match.off = (int)(ip-matchPtr); + return match; +} + + +static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, + const char* const source, + char* dst, + int* srcSizePtr, + int dstCapacity, + int const nbSearches, + size_t sufficient_len, + const limitedOutput_directive limit, + int const fullUpdate, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) +{ +#define TRAILING_LITERALS 3 + LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which is a bit large for stack... */ + + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + BYTE* op = (BYTE*) dst; + BYTE* opSaved = (BYTE*) dst; + BYTE* oend = op + dstCapacity; + + /* init */ + DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity); + *srcSizePtr = 0; + if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; + + /* Main Loop */ + assert(ip - anchor < LZ4_MAX_INPUT_SIZE); + while (ip <= mflimit) { + int const llen = (int)(ip - anchor); + int best_mlen, best_off; + int cur, last_match_pos = 0; + + LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed); + if (firstMatch.len==0) { ip++; continue; } + + if ((size_t)firstMatch.len > sufficient_len) { + /* good enough solution : immediate encoding */ + int const firstML = firstMatch.len; + const BYTE* const matchPos = ip - firstMatch.off; + opSaved = op; + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */ + goto _dest_overflow; + continue; + } + + /* set prices for first positions (literals) */ + { int rPos; + for (rPos = 0 ; rPos < MINMATCH ; rPos++) { + int const cost = LZ4HC_literalsPrice(llen + rPos); + opt[rPos].mlen = 1; + opt[rPos].off = 0; + opt[rPos].litlen = llen + rPos; + opt[rPos].price = cost; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", + rPos, cost, opt[rPos].litlen); + } } + /* set prices using initial match */ + { int mlen = MINMATCH; + int const matchML = firstMatch.len; /* necessarily < sufficient_len < LZ4_OPT_NUM */ + int const offset = firstMatch.off; + assert(matchML < LZ4_OPT_NUM); + for ( ; mlen <= matchML ; mlen++) { + int const cost = LZ4HC_sequencePrice(llen, mlen); + opt[mlen].mlen = mlen; + opt[mlen].off = offset; + opt[mlen].litlen = llen; + opt[mlen].price = cost; + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup", + mlen, cost, mlen); + } } + last_match_pos = firstMatch.len; + { int addLit; + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { + opt[last_match_pos+addLit].mlen = 1; /* literal */ + opt[last_match_pos+addLit].off = 0; + opt[last_match_pos+addLit].litlen = addLit; + opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", + last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); + } } + + /* check further positions */ + for (cur = 1; cur < last_match_pos; cur++) { + const BYTE* const curPtr = ip + cur; + LZ4HC_match_t newMatch; + + if (curPtr > mflimit) break; + DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u", + cur, opt[cur].price, opt[cur+1].price, cur+1); + if (fullUpdate) { + /* not useful to search here if next position has same (or lower) cost */ + if ( (opt[cur+1].price <= opt[cur].price) + /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */ + && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) ) + continue; + } else { + /* not useful to search here if next position has same (or lower) cost */ + if (opt[cur+1].price <= opt[cur].price) continue; + } + + DEBUGLOG(7, "search at rPos:%u", cur); + if (fullUpdate) + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed); + else + /* only test matches of minimum length; slightly faster, but misses a few bytes */ + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict, favorDecSpeed); + if (!newMatch.len) continue; + + if ( ((size_t)newMatch.len > sufficient_len) + || (newMatch.len + cur >= LZ4_OPT_NUM) ) { + /* immediate encoding */ + best_mlen = newMatch.len; + best_off = newMatch.off; + last_match_pos = cur + 1; + goto encode; + } + + /* before match : set price with literals at beginning */ + { int const baseLitlen = opt[cur].litlen; + int litlen; + for (litlen = 1; litlen < MINMATCH; litlen++) { + int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen); + int const pos = cur + litlen; + if (price < opt[pos].price) { + opt[pos].mlen = 1; /* literal */ + opt[pos].off = 0; + opt[pos].litlen = baseLitlen+litlen; + opt[pos].price = price; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", + pos, price, opt[pos].litlen); + } } } + + /* set prices using match at position = cur */ + { int const matchML = newMatch.len; + int ml = MINMATCH; + + assert(cur + newMatch.len < LZ4_OPT_NUM); + for ( ; ml <= matchML ; ml++) { + int const pos = cur + ml; + int const offset = newMatch.off; + int price; + int ll; + DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)", + pos, last_match_pos); + if (opt[cur].mlen == 1) { + ll = opt[cur].litlen; + price = ((cur > ll) ? opt[cur - ll].price : 0) + + LZ4HC_sequencePrice(ll, ml); + } else { + ll = 0; + price = opt[cur].price + LZ4HC_sequencePrice(0, ml); + } + + assert((U32)favorDecSpeed <= 1); + if (pos > last_match_pos+TRAILING_LITERALS + || price <= opt[pos].price - (int)favorDecSpeed) { + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)", + pos, price, ml); + assert(pos < LZ4_OPT_NUM); + if ( (ml == matchML) /* last pos of last match */ + && (last_match_pos < pos) ) + last_match_pos = pos; + opt[pos].mlen = ml; + opt[pos].off = offset; + opt[pos].litlen = ll; + opt[pos].price = price; + } } } + /* complete following positions with literals */ + { int addLit; + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { + opt[last_match_pos+addLit].mlen = 1; /* literal */ + opt[last_match_pos+addLit].off = 0; + opt[last_match_pos+addLit].litlen = addLit; + opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); + } } + } /* for (cur = 1; cur <= last_match_pos; cur++) */ + + assert(last_match_pos < LZ4_OPT_NUM + TRAILING_LITERALS); + best_mlen = opt[last_match_pos].mlen; + best_off = opt[last_match_pos].off; + cur = last_match_pos - best_mlen; + + encode: /* cur, last_match_pos, best_mlen, best_off must be set */ + assert(cur < LZ4_OPT_NUM); + assert(last_match_pos >= 1); /* == 1 when only one candidate */ + DEBUGLOG(6, "reverse traversal, looking for shortest path (last_match_pos=%i)", last_match_pos); + { int candidate_pos = cur; + int selected_matchLength = best_mlen; + int selected_offset = best_off; + while (1) { /* from end to beginning */ + int const next_matchLength = opt[candidate_pos].mlen; /* can be 1, means literal */ + int const next_offset = opt[candidate_pos].off; + DEBUGLOG(7, "pos %i: sequence length %i", candidate_pos, selected_matchLength); + opt[candidate_pos].mlen = selected_matchLength; + opt[candidate_pos].off = selected_offset; + selected_matchLength = next_matchLength; + selected_offset = next_offset; + if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */ + assert(next_matchLength > 0); /* can be 1, means literal */ + candidate_pos -= next_matchLength; + } } + + /* encode all recorded sequences in order */ + { int rPos = 0; /* relative position (to ip) */ + while (rPos < last_match_pos) { + int const ml = opt[rPos].mlen; + int const offset = opt[rPos].off; + if (ml == 1) { ip++; rPos++; continue; } /* literal; note: can end up with several literals, in which case, skip them */ + rPos += ml; + assert(ml >= MINMATCH); + assert((offset >= 1) && (offset <= LZ4_DISTANCE_MAX)); + opSaved = op; + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */ + goto _dest_overflow; + } } + } /* while (ip <= mflimit) */ + + _last_literals: + /* Encode Last Literals */ + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + litLength + lastRunSize; + if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ + if (limit && (op + totalSize > oend)) { + if (limit == limitedOutput) return 0; /* Check output limit */ + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (size_t)(oend - op) - 1; + litLength = (lastRunSize + 255 - RUN_MASK) / 255; + lastRunSize -= litLength; + } + ip = anchor + lastRunSize; + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + memcpy(op, anchor, lastRunSize); + op += lastRunSize; + } + + /* End */ + *srcSizePtr = (int) (((const char*)ip) - source); + return (int) ((char*)op-dst); + + _dest_overflow: + if (limit == fillOutput) { + op = opSaved; /* restore correct out pointer */ + goto _last_literals; + } + return 0; + } diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.h index deb2394062b81..44e35bbf6b7dc 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.h @@ -1,7 +1,7 @@ /* LZ4 HC - High Compression Mode of LZ4 Header File - Copyright (C) 2011-2014, Yann Collet. + Copyright (C) 2011-2017, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -28,146 +28,411 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ -#pragma once - +#ifndef LZ4_HC_H_19834876238432 +#define LZ4_HC_H_19834876238432 #if defined (__cplusplus) extern "C" { #endif - -int LZ4_compressHC (const char* source, char* dest, int inputSize); -/* -LZ4_compressHC : - return : the number of bytes in compressed buffer dest - or 0 if compression fails. - note : destination buffer must be already allocated. - To avoid any problem, size it to handle worst cases situations (input data not compressible) - Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h") -*/ - -int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); -/* -LZ4_compress_limitedOutput() : - Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. - If it cannot achieve it, compression will stop, and result of the function will be zero. - This function never writes outside of provided output buffer. - - inputSize : Max supported value is 1 GB - maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated) - return : the number of output bytes written in buffer 'dest' - or 0 if compression fails. -*/ +/* --- Dependency --- */ +/* note : lz4hc requires lz4.h/lz4.c for compilation */ +#include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */ -int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); -int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -/* - Same functions as above, but with programmable 'compressionLevel'. - Recommended values are between 4 and 9, although any value between 0 and 16 will work. - 'compressionLevel'==0 means use default 'compressionLevel' value. - Values above 16 behave the same as 16. - Equivalent variants exist for all other compression functions below. -*/ +/* --- Useful constants --- */ +#define LZ4HC_CLEVEL_MIN 3 +#define LZ4HC_CLEVEL_DEFAULT 9 +#define LZ4HC_CLEVEL_OPT_MIN 10 +#define LZ4HC_CLEVEL_MAX 12 -/* Note : -Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license) -*/ +/*-************************************ + * Block Compression + **************************************/ +/*! LZ4_compress_HC() : + * Compress data from `src` into `dst`, using the powerful but slower "HC" algorithm. + * `dst` must be already allocated. + * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") + * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") + * `compressionLevel` : any value between 1 and LZ4HC_CLEVEL_MAX will work. + * Values > LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX. + * @return : the number of bytes written into 'dst' + * or 0 if compression fails. + */ +LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); -/************************************** - Using an external allocation -**************************************/ -int LZ4_sizeofStateHC(void); -int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); -int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); -int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +/* Note : + * Decompression functions are provided within "lz4.h" (BSD license) + */ + + +/*! LZ4_compress_HC_extStateHC() : + * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`. + * `state` size is provided by LZ4_sizeofStateHC(). + * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() should do properly). + */ +LZ4LIB_API int LZ4_sizeofStateHC(void); +LZ4LIB_API int LZ4_compress_HC_extStateHC(void* stateHC, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); + + +/*! LZ4_compress_HC_destSize() : v1.9.0+ + * Will compress as much data as possible from `src` + * to fit into `targetDstSize` budget. + * Result is provided in 2 parts : + * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize) + * or 0 if compression fails. + * `srcSizePtr` : on success, *srcSizePtr is updated to indicate how much bytes were read from `src` + */ +LZ4LIB_API int LZ4_compress_HC_destSize(void* stateHC, + const char* src, char* dst, + int* srcSizePtr, int targetDstSize, + int compressionLevel); + + +/*-************************************ + * Streaming Compression + * Bufferless synchronous API + **************************************/ + typedef union LZ4_streamHC_u LZ4_streamHC_t; /* incomplete type (defined later) */ + +/*! LZ4_createStreamHC() and LZ4_freeStreamHC() : + * These functions create and release memory for LZ4 HC streaming state. + * Newly created states are automatically initialized. + * A same state can be used multiple times consecutively, + * starting with LZ4_resetStreamHC_fast() to start a new stream of blocks. + */ +LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); +LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); /* -These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods. -To know how much memory must be allocated for the compression tables, use : -int LZ4_sizeofStateHC(); + These functions compress data in successive blocks of any size, + using previous blocks as dictionary, to improve compression ratio. + One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. + There is an exception for ring buffers, which can be smaller than 64 KB. + Ring-buffer scenario is automatically detected and handled within LZ4_compress_HC_continue(). + + Before starting compression, state must be allocated and properly initialized. + LZ4_createStreamHC() does both, though compression level is set to LZ4HC_CLEVEL_DEFAULT. + + Selecting the compression level can be done with LZ4_resetStreamHC_fast() (starts a new stream) + or LZ4_setCompressionLevel() (anytime, between blocks in the same stream) (experimental). + LZ4_resetStreamHC_fast() only works on states which have been properly initialized at least once, + which is automatically the case when state is created using LZ4_createStreamHC(). + + After reset, a first "fictional block" can be designated as initial dictionary, + using LZ4_loadDictHC() (Optional). + + Invoke LZ4_compress_HC_continue() to compress each successive block. + The number of blocks is unlimited. + Previous input blocks, including initial dictionary when present, + must remain accessible and unmodified during compression. + + It's allowed to update compression level anytime between blocks, + using LZ4_setCompressionLevel() (experimental). + + 'dst' buffer should be sized to handle worst case scenarios + (see LZ4_compressBound(), it ensures compression success). + In case of failure, the API does not guarantee recovery, + so the state _must_ be reset. + To ensure compression success + whenever `dst` buffer size cannot be made >= LZ4_compressBound(), + consider using LZ4_compress_HC_continue_destSize(). + + Whenever previous input blocks can't be preserved unmodified in-place during compression of next blocks, + it's possible to copy the last blocks into a more stable memory space, using LZ4_saveDictHC(). + Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer' (<= 64 KB) + + After completing a streaming compression, + it's possible to start a new stream of blocks, using the same LZ4_streamHC_t state, + just by resetting it, using LZ4_resetStreamHC_fast(). +*/ -Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0). +LZ4LIB_API void LZ4_resetStreamHC_fast(LZ4_streamHC_t* streamHCPtr, int compressionLevel); /* v1.9.0+ */ +LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); + +LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, + const char* src, char* dst, + int srcSize, int maxDstSize); + +/*! LZ4_compress_HC_continue_destSize() : v1.9.0+ + * Similar to LZ4_compress_HC_continue(), + * but will read as much data as possible from `src` + * to fit into `targetDstSize` budget. + * Result is provided into 2 parts : + * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize) + * or 0 if compression fails. + * `srcSizePtr` : on success, *srcSizePtr will be updated to indicate how much bytes were read from `src`. + * Note that this function may not consume the entire input. + */ +LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, + const char* src, char* dst, + int* srcSizePtr, int targetDstSize); + +LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); + + + +/*^********************************************** + * !!!!!! STATIC LINKING ONLY !!!!!! + ***********************************************/ + +/*-****************************************************************** + * PRIVATE DEFINITIONS : + * Do not use these definitions directly. + * They are merely exposed to allow static allocation of `LZ4_streamHC_t`. + * Declare an `LZ4_streamHC_t` directly, rather than any type below. + * Even then, only do so in the context of static linking, as definitions may change between versions. + ********************************************************************/ + +#define LZ4HC_DICTIONARY_LOGSIZE 16 +#define LZ4HC_MAXD (1<= 199901L) /* C99 */) +#include + +typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal; +struct LZ4HC_CCtx_internal +{ + uint32_t hashTable[LZ4HC_HASHTABLESIZE]; + uint16_t chainTable[LZ4HC_MAXD]; + const uint8_t* end; /* next block here to continue on current prefix */ + const uint8_t* base; /* All index relative to this position */ + const uint8_t* dictBase; /* alternate base for extDict */ + uint32_t dictLimit; /* below that point, need extDict */ + uint32_t lowLimit; /* below that point, no more dict */ + uint32_t nextToUpdate; /* index from which to continue dictionary update */ + short compressionLevel; + int8_t favorDecSpeed; /* favor decompression speed if this flag set, + otherwise, favor compression ratio */ + int8_t dirty; /* stream has to be fully reset if this flag is set */ + const LZ4HC_CCtx_internal* dictCtx; +}; + +#else + +typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal; +struct LZ4HC_CCtx_internal +{ + unsigned int hashTable[LZ4HC_HASHTABLESIZE]; + unsigned short chainTable[LZ4HC_MAXD]; + const unsigned char* end; /* next block here to continue on current prefix */ + const unsigned char* base; /* All index relative to this position */ + const unsigned char* dictBase; /* alternate base for extDict */ + unsigned int dictLimit; /* below that point, need extDict */ + unsigned int lowLimit; /* below that point, no more dict */ + unsigned int nextToUpdate; /* index from which to continue dictionary update */ + short compressionLevel; + char favorDecSpeed; /* favor decompression speed if this flag set, + otherwise, favor compression ratio */ + char dirty; /* stream has to be fully reset if this flag is set */ + const LZ4HC_CCtx_internal* dictCtx; +}; -The allocated memory can be provided to the compressions functions using 'void* state' parameter. -LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions. -They just use the externally allocated memory area instead of allocating their own (on stack, or on heap). -*/ +#endif -/************************************** - Streaming Functions +/* Do not use these definitions directly ! + * Declare or allocate an LZ4_streamHC_t instead. + */ +#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56 + ((sizeof(void*)==16) ? 56 : 0) /* AS400*/ ) /* 262200 or 262256*/ +#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) +union LZ4_streamHC_u { + size_t table[LZ4_STREAMHCSIZE_SIZET]; + LZ4HC_CCtx_internal internal_donotuse; +}; /* previously typedef'd to LZ4_streamHC_t */ + +/* LZ4_streamHC_t : + * This structure allows static allocation of LZ4 HC streaming state. + * This can be used to allocate statically, on state, or as part of a larger structure. + * + * Such state **must** be initialized using LZ4_initStreamHC() before first use. + * + * Note that invoking LZ4_initStreamHC() is not required when + * the state was created using LZ4_createStreamHC() (which is recommended). + * Using the normal builder, a newly created state is automatically initialized. + * + * Static allocation shall only be used in combination with static linking. + */ + +/* LZ4_initStreamHC() : v1.9.0+ + * Required before first use of a statically allocated LZ4_streamHC_t. + * Before v1.9.0 : use LZ4_resetStreamHC() instead + */ +LZ4LIB_API LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size); + + +/*-************************************ +* Deprecated Functions **************************************/ -/* Note : these streaming functions still follows the older model */ -void* LZ4_createHC (const char* inputBuffer); -int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize); -int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize); -char* LZ4_slideInputBufferHC (void* LZ4HC_Data); -int LZ4_freeHC (void* LZ4HC_Data); +/* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */ + +/* deprecated compression functions */ +LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC (const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/* Obsolete streaming functions; degraded functionality; do not use! + * + * In order to perform streaming compression, these functions depended on data + * that is no longer tracked in the state. They have been preserved as well as + * possible: using them will still produce a correct output. However, use of + * LZ4_slideInputBufferHC() will truncate the history of the stream, rather + * than preserve a window-sized chunk of history. + */ +LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer); +LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data); +LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API int LZ4_sizeofStreamStateHC(void); +LZ4_DEPRECATED("use LZ4_initStreamHC() instead") LZ4LIB_API int LZ4_resetStreamStateHC(void* state, char* inputBuffer); + + +/* LZ4_resetStreamHC() is now replaced by LZ4_initStreamHC(). + * The intention is to emphasize the difference with LZ4_resetStreamHC_fast(), + * which is now the recommended function to start a new stream of blocks, + * but cannot be used to initialize a memory segment containing arbitrary garbage data. + * + * It is recommended to switch to LZ4_initStreamHC(). + * LZ4_resetStreamHC() will generate deprecation warnings in a future version. + */ +LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); -int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); -int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -/* -These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks. -In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function : - -void* LZ4_createHC (const char* inputBuffer); -The result of the function is the (void*) pointer on the LZ4HC Data Structure. -This pointer will be needed in all other functions. -If the pointer returned is NULL, then the allocation has failed, and compression must be aborted. -The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. -The input buffer must be already allocated, and size at least 192KB. -'inputBuffer' will also be the 'const char* source' of the first block. - -All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'. -To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(). -Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(), -but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one. -If next block does not begin immediately after the previous one, the compression will fail (return 0). - -When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to : -char* LZ4_slideInputBufferHC(void* LZ4HC_Data); -must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer. -Note that, for this function to work properly, minimum size of an input buffer must be 192KB. -==> The memory position where the next input data block must start is provided as the result of the function. - -Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual. - -When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure. -*/ - -int LZ4_sizeofStreamStateHC(void); -int LZ4_resetStreamStateHC(void* state, const char* inputBuffer); +#if defined (__cplusplus) +} +#endif -/* -These functions achieve the same result as : -void* LZ4_createHC (const char* inputBuffer); +#endif /* LZ4_HC_H_19834876238432 */ -They are provided here to allow the user program to allocate memory using its own routines. -To know how much space must be allocated, use LZ4_sizeofStreamStateHC(); -Note also that space must be aligned for pointers (32 or 64 bits). +/*-************************************************** + * !!!!! STATIC LINKING ONLY !!!!! + * Following definitions are considered experimental. + * They should not be linked from DLL, + * as there is no guarantee of API stability yet. + * Prototypes will be promoted to "stable" status + * after successfull usage in real-life scenarios. + ***************************************************/ +#ifdef LZ4_HC_STATIC_LINKING_ONLY /* protection macro */ +#ifndef LZ4_HC_SLO_098092834 +#define LZ4_HC_SLO_098092834 -Once space is allocated, you must initialize it using : LZ4_resetStreamStateHC(void* state, const char* inputBuffer); -void* state is a pointer to the space allocated. -It must be aligned for pointers (32 or 64 bits), and be large enough. -The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. -The input buffer must be already allocated, and size at least 192KB. -'inputBuffer' will also be the 'const char* source' of the first block. +#define LZ4_STATIC_LINKING_ONLY /* LZ4LIB_STATIC_API */ +#include "lz4.h" -The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState(). -return value of LZ4_resetStreamStateHC() must be 0 is OK. -Any other value means there was an error (typically, state is not aligned for pointers (32 or 64 bits)). -*/ +#if defined (__cplusplus) +extern "C" { +#endif +/*! LZ4_setCompressionLevel() : v1.8.0+ (experimental) + * It's possible to change compression level + * between successive invocations of LZ4_compress_HC_continue*() + * for dynamic adaptation. + */ +LZ4LIB_STATIC_API void LZ4_setCompressionLevel( + LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); + +/*! LZ4_favorDecompressionSpeed() : v1.8.2+ (experimental) + * Opt. Parser will favor decompression speed over compression ratio. + * Only applicable to levels >= LZ4HC_CLEVEL_OPT_MIN. + */ +LZ4LIB_STATIC_API void LZ4_favorDecompressionSpeed( + LZ4_streamHC_t* LZ4_streamHCPtr, int favor); + +/*! LZ4_resetStreamHC_fast() : v1.9.0+ + * When an LZ4_streamHC_t is known to be in a internally coherent state, + * it can often be prepared for a new compression with almost no work, only + * sometimes falling back to the full, expensive reset that is always required + * when the stream is in an indeterminate state (i.e., the reset performed by + * LZ4_resetStreamHC()). + * + * LZ4_streamHCs are guaranteed to be in a valid state when: + * - returned from LZ4_createStreamHC() + * - reset by LZ4_resetStreamHC() + * - memset(stream, 0, sizeof(LZ4_streamHC_t)) + * - the stream was in a valid state and was reset by LZ4_resetStreamHC_fast() + * - the stream was in a valid state and was then used in any compression call + * that returned success + * - the stream was in an indeterminate state and was used in a compression + * call that fully reset the state (LZ4_compress_HC_extStateHC()) and that + * returned success + * + * Note: + * A stream that was last used in a compression call that returned an error + * may be passed to this function. However, it will be fully reset, which will + * clear any existing history and settings from the context. + */ +LZ4LIB_STATIC_API void LZ4_resetStreamHC_fast( + LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); + +/*! LZ4_compress_HC_extStateHC_fastReset() : + * A variant of LZ4_compress_HC_extStateHC(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStreamHC_fast() for a definition of + * "correctly initialized"). From a high level, the difference is that this + * function initializes the provided state with a call to + * LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a + * call to LZ4_resetStreamHC(). + */ +LZ4LIB_STATIC_API int LZ4_compress_HC_extStateHC_fastReset ( + void* state, + const char* src, char* dst, + int srcSize, int dstCapacity, + int compressionLevel); + +/*! LZ4_attach_HC_dictionary() : + * This is an experimental API that allows for the efficient use of a + * static dictionary many times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a + * working LZ4_streamHC_t, this function introduces a no-copy setup mechanism, + * in which the working stream references the dictionary stream in-place. + * + * Several assumptions are made about the state of the dictionary stream. + * Currently, only streams which have been prepared by LZ4_loadDictHC() should + * be expected to work. + * + * Alternatively, the provided dictionary stream pointer may be NULL, in which + * case any existing dictionary stream is unset. + * + * A dictionary should only be attached to a stream without any history (i.e., + * a stream that has just been reset). + * + * The dictionary will remain attached to the working stream only for the + * current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the + * dictionary context association from the working stream. The dictionary + * stream (and source buffer) must remain in-place / accessible / unchanged + * through the lifetime of the stream session. + */ +LZ4LIB_STATIC_API void LZ4_attach_HC_dictionary( + LZ4_streamHC_t *working_stream, + const LZ4_streamHC_t *dictionary_stream); #if defined (__cplusplus) } #endif + +#endif /* LZ4_HC_SLO_098092834 */ +#endif /* LZ4_HC_STATIC_LINKING_ONLY */ diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c deleted file mode 100644 index 9a09f078d8260..0000000000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include "org_apache_hadoop_io_compress_snappy.h" - -#if defined HADOOP_SNAPPY_LIBRARY - -#include -#include -#include - -#ifdef UNIX -#include -#include "config.h" -#endif // UNIX - -#ifdef WINDOWS -#include "winutils.h" -#endif - -#include "org_apache_hadoop_io_compress_snappy_SnappyCompressor.h" - -#define JINT_MAX 0x7fffffff - -static jfieldID SnappyCompressor_uncompressedDirectBuf; -static jfieldID SnappyCompressor_uncompressedDirectBufLen; -static jfieldID SnappyCompressor_compressedDirectBuf; -static jfieldID SnappyCompressor_directBufferSize; - -#ifdef UNIX -static snappy_status (*dlsym_snappy_compress)(const char*, size_t, char*, size_t*); -#endif - -#ifdef WINDOWS -typedef snappy_status (__cdecl *__dlsym_snappy_compress)(const char*, size_t, char*, size_t*); -static __dlsym_snappy_compress dlsym_snappy_compress; -#endif - -JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompressor_initIDs -(JNIEnv *env, jclass clazz){ -#ifdef UNIX - // Load libsnappy.so - void *libsnappy = dlopen(HADOOP_SNAPPY_LIBRARY, RTLD_LAZY | RTLD_GLOBAL); - if (!libsnappy) { - char msg[1000]; - snprintf(msg, 1000, "%s (%s)!", "Cannot load " HADOOP_SNAPPY_LIBRARY, dlerror()); - THROW(env, "java/lang/UnsatisfiedLinkError", msg); - return; - } -#endif - -#ifdef WINDOWS - HMODULE libsnappy = LoadLibrary(HADOOP_SNAPPY_LIBRARY); - if (!libsnappy) { - THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load snappy.dll"); - return; - } -#endif - - // Locate the requisite symbols from libsnappy.so -#ifdef UNIX - dlerror(); // Clear any existing error - LOAD_DYNAMIC_SYMBOL(dlsym_snappy_compress, env, libsnappy, "snappy_compress"); -#endif - -#ifdef WINDOWS - LOAD_DYNAMIC_SYMBOL(__dlsym_snappy_compress, dlsym_snappy_compress, env, libsnappy, "snappy_compress"); -#endif - - SnappyCompressor_uncompressedDirectBuf = (*env)->GetFieldID(env, clazz, - "uncompressedDirectBuf", - "Ljava/nio/Buffer;"); - SnappyCompressor_uncompressedDirectBufLen = (*env)->GetFieldID(env, clazz, - "uncompressedDirectBufLen", "I"); - SnappyCompressor_compressedDirectBuf = (*env)->GetFieldID(env, clazz, - "compressedDirectBuf", - "Ljava/nio/Buffer;"); - SnappyCompressor_directBufferSize = (*env)->GetFieldID(env, clazz, - "directBufferSize", "I"); -} - -JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompressor_compressBytesDirect -(JNIEnv *env, jobject thisj){ - const char* uncompressed_bytes; - char* compressed_bytes; - snappy_status ret; - // Get members of SnappyCompressor - jobject uncompressed_direct_buf = (*env)->GetObjectField(env, thisj, SnappyCompressor_uncompressedDirectBuf); - jint uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, SnappyCompressor_uncompressedDirectBufLen); - jobject compressed_direct_buf = (*env)->GetObjectField(env, thisj, SnappyCompressor_compressedDirectBuf); - jint compressed_direct_buf_len = (*env)->GetIntField(env, thisj, SnappyCompressor_directBufferSize); - size_t buf_len; - - // Get the input direct buffer - uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); - - if (uncompressed_bytes == 0) { - return (jint)0; - } - - // Get the output direct buffer - compressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf); - - if (compressed_bytes == 0) { - return (jint)0; - } - - /* size_t should always be 4 bytes or larger. */ - buf_len = (size_t)compressed_direct_buf_len; - ret = dlsym_snappy_compress(uncompressed_bytes, uncompressed_direct_buf_len, - compressed_bytes, &buf_len); - if (ret != SNAPPY_OK){ - THROW(env, "java/lang/InternalError", "Could not compress data. Buffer length is too small."); - return 0; - } - if (buf_len > JINT_MAX) { - THROW(env, "java/lang/InternalError", "Invalid return buffer length."); - return 0; - } - - (*env)->SetIntField(env, thisj, SnappyCompressor_uncompressedDirectBufLen, 0); - return (jint)buf_len; -} - -JNIEXPORT jstring JNICALL -Java_org_apache_hadoop_io_compress_snappy_SnappyCompressor_getLibraryName(JNIEnv *env, jclass class) { -#ifdef UNIX - if (dlsym_snappy_compress) { - Dl_info dl_info; - if(dladdr( - dlsym_snappy_compress, - &dl_info)) { - return (*env)->NewStringUTF(env, dl_info.dli_fname); - } - } - - return (*env)->NewStringUTF(env, HADOOP_SNAPPY_LIBRARY); -#endif - -#ifdef WINDOWS - LPWSTR filename = NULL; - GetLibraryName(dlsym_snappy_compress, &filename); - if (filename != NULL) { - return (*env)->NewString(env, filename, (jsize) wcslen(filename)); - } else { - return (*env)->NewStringUTF(env, "Unavailable"); - } -#endif -} -#endif //define HADOOP_SNAPPY_LIBRARY diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c deleted file mode 100644 index 69ec1017526fd..0000000000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "org_apache_hadoop_io_compress_snappy.h" - -#if defined HADOOP_SNAPPY_LIBRARY - -#include -#include -#include - -#ifdef UNIX -#include -#include "config.h" -#endif - -#include "org_apache_hadoop_io_compress_snappy_SnappyDecompressor.h" - -static jfieldID SnappyDecompressor_compressedDirectBuf; -static jfieldID SnappyDecompressor_compressedDirectBufLen; -static jfieldID SnappyDecompressor_uncompressedDirectBuf; -static jfieldID SnappyDecompressor_directBufferSize; - -#ifdef UNIX -static snappy_status (*dlsym_snappy_uncompress)(const char*, size_t, char*, size_t*); -#endif - -#ifdef WINDOWS -typedef snappy_status (__cdecl *__dlsym_snappy_uncompress)(const char*, size_t, char*, size_t*); -static __dlsym_snappy_uncompress dlsym_snappy_uncompress; -#endif - -JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyDecompressor_initIDs -(JNIEnv *env, jclass clazz){ - - // Load libsnappy.so -#ifdef UNIX - void *libsnappy = dlopen(HADOOP_SNAPPY_LIBRARY, RTLD_LAZY | RTLD_GLOBAL); - if (!libsnappy) { - char* msg = (char*)malloc(1000); - snprintf(msg, 1000, "%s (%s)!", "Cannot load " HADOOP_SNAPPY_LIBRARY, dlerror()); - THROW(env, "java/lang/UnsatisfiedLinkError", msg); - return; - } -#endif - -#ifdef WINDOWS - HMODULE libsnappy = LoadLibrary(HADOOP_SNAPPY_LIBRARY); - if (!libsnappy) { - THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load snappy.dll"); - return; - } -#endif - - // Locate the requisite symbols from libsnappy.so -#ifdef UNIX - dlerror(); // Clear any existing error - LOAD_DYNAMIC_SYMBOL(dlsym_snappy_uncompress, env, libsnappy, "snappy_uncompress"); - -#endif - -#ifdef WINDOWS - LOAD_DYNAMIC_SYMBOL(__dlsym_snappy_uncompress, dlsym_snappy_uncompress, env, libsnappy, "snappy_uncompress"); -#endif - - SnappyDecompressor_compressedDirectBuf = (*env)->GetFieldID(env,clazz, - "compressedDirectBuf", - "Ljava/nio/Buffer;"); - SnappyDecompressor_compressedDirectBufLen = (*env)->GetFieldID(env,clazz, - "compressedDirectBufLen", "I"); - SnappyDecompressor_uncompressedDirectBuf = (*env)->GetFieldID(env,clazz, - "uncompressedDirectBuf", - "Ljava/nio/Buffer;"); - SnappyDecompressor_directBufferSize = (*env)->GetFieldID(env, clazz, - "directBufferSize", "I"); -} - -JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyDecompressor_decompressBytesDirect -(JNIEnv *env, jobject thisj){ - const char* compressed_bytes = NULL; - char* uncompressed_bytes = NULL; - snappy_status ret; - // Get members of SnappyDecompressor - jobject compressed_direct_buf = (*env)->GetObjectField(env,thisj, SnappyDecompressor_compressedDirectBuf); - jint compressed_direct_buf_len = (*env)->GetIntField(env,thisj, SnappyDecompressor_compressedDirectBufLen); - jobject uncompressed_direct_buf = (*env)->GetObjectField(env,thisj, SnappyDecompressor_uncompressedDirectBuf); - size_t uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, SnappyDecompressor_directBufferSize); - - // Get the input direct buffer - compressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf); - - if (compressed_bytes == 0) { - return (jint)0; - } - - // Get the output direct buffer - uncompressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); - - if (uncompressed_bytes == 0) { - return (jint)0; - } - - ret = dlsym_snappy_uncompress(compressed_bytes, compressed_direct_buf_len, - uncompressed_bytes, &uncompressed_direct_buf_len); - if (ret == SNAPPY_BUFFER_TOO_SMALL){ - THROW(env, "java/lang/InternalError", "Could not decompress data. Buffer length is too small."); - } else if (ret == SNAPPY_INVALID_INPUT){ - THROW(env, "java/lang/InternalError", "Could not decompress data. Input is invalid."); - } else if (ret != SNAPPY_OK){ - THROW(env, "java/lang/InternalError", "Could not decompress data."); - } - - (*env)->SetIntField(env, thisj, SnappyDecompressor_compressedDirectBufLen, 0); - - return (jint)uncompressed_direct_buf_len; -} - -#endif //define HADOOP_SNAPPY_LIBRARY diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c index 41eb9e2c85a10..6581f292b4a00 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c @@ -219,13 +219,13 @@ JNIEXPORT jint Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_defla return (jint) 0; } - bytes_read += input.pos; + bytes_read += input.pos - uncompressed_direct_buf_off; bytes_written += output.pos; (*env)->SetLongField(env, this, ZStandardCompressor_bytesRead, bytes_read); (*env)->SetLongField(env, this, ZStandardCompressor_bytesWritten, bytes_written); (*env)->SetIntField(env, this, ZStandardCompressor_uncompressedDirectBufOff, input.pos); - (*env)->SetIntField(env, this, ZStandardCompressor_uncompressedDirectBufLen, input.size - input.pos); + (*env)->SetIntField(env, this, ZStandardCompressor_uncompressedDirectBufLen, input.size); return (jint) output.pos; } diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c index 9cca6dd754b09..816536b637d39 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c @@ -92,6 +92,7 @@ void getInputs(JNIEnv *env, jobjectArray inputs, jintArray inputOffsets, destInputs[i] = NULL; } } + (*env)->ReleaseIntArrayElements(env, inputOffsets, tmpInputOffsets, 0); } void getOutputs(JNIEnv *env, jobjectArray outputs, jintArray outputOffsets, @@ -112,4 +113,5 @@ void getOutputs(JNIEnv *env, jobjectArray outputs, jintArray outputOffsets, byteBuffer)); destOutputs[i] += tmpOutputOffsets[i]; } -} \ No newline at end of file + (*env)->ReleaseIntArrayElements(env, outputOffsets, tmpOutputOffsets, 0); +} diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c index 52d255afd58d8..72314d2ad545a 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c @@ -66,6 +66,7 @@ jintArray outputOffsets) { decode(&rsDecoder->decoder, rsDecoder->inputs, tmpErasedIndexes, numErased, rsDecoder->outputs, chunkSize); + (*env)->ReleaseIntArrayElements(env, erasedIndexes, tmpErasedIndexes, 0); } JNIEXPORT void JNICALL diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory index f673cf4cae427..1c6fc74d33406 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory +++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory @@ -16,3 +16,5 @@ org.apache.hadoop.security.alias.JavaKeyStoreProvider$Factory org.apache.hadoop.security.alias.LocalJavaKeyStoreProvider$Factory org.apache.hadoop.security.alias.UserProvider$Factory +org.apache.hadoop.security.alias.BouncyCastleFipsKeyStoreProvider$Factory +org.apache.hadoop.security.alias.LocalBouncyCastleFipsKeyStoreProvider$Factory diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index cf156af196461..8fcdd40bd6dc6 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -2584,6 +2584,15 @@ + + ipc.[port_number].decay-scheduler.service-users + + Service users will always be scheduled into the highest-priority + queue and won't be included in the priority computation of normal user + calls. They are specified as a comma-separated list. + + + ipc.[port_number].weighted-cost.lockshared 10 @@ -3300,6 +3309,26 @@ + + hadoop.security.crypto.codec.classes.sm4.ctr.nopadding + org.apache.hadoop.crypto.OpensslSm4CtrCryptoCodec, org.apache.hadoop.crypto.JceSm4CtrCryptoCodec + + Comma-separated list of crypto codec implementations for SM4/CTR/NoPadding. + The first implementation will be used if available, others are fallbacks. + + + + + hadoop.security.openssl.engine.id + + + The Openssl provided an engine mechanism that allow to specify third-party software + encryption library or hardware encryption device for encryption. The engine ID could + be vendor defined and will be passed to openssl, more info please see: + https://github.com/openssl/openssl/blob/master/README.ENGINE + + + hadoop.security.crypto.cipher.suite AES/CTR/NoPadding @@ -3818,6 +3847,16 @@ in audit logs. + + hadoop.caller.context.separator + , + + The separator is for context which maybe contain many fields. For example, + if the separator is ',', and there are two key/value fields in context, + in which case the context string is "key1:value1,key2:value2". The + separator should not contain '\t', '\n', '='. + + seq.io.sort.mb @@ -3979,4 +4018,18 @@ ThreadGroup approach is preferred for better performance. + + + fs.AbstractFileSystem.ofs.impl + org.apache.hadoop.fs.ozone.RootedOzFs + The AbstractFileSystem for Rooted Ozone + FileSystem ofs uri + + + + fs.AbstractFileSystem.o3fs.impl + org.apache.hadoop.fs.ozone.OzFs + The AbstractFileSystem for Ozone FileSystem o3fs uri + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md index 0bda253fc8b54..4842d5b86d621 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md @@ -60,7 +60,7 @@ Many subcommands honor a common set of configuration options to alter their beha | `-files ` | Specify comma separated files to be copied to the map reduce cluster. Applies only to job. | | `-fs or ` | Specify default filesystem URL to use. Overrides 'fs.defaultFS' property from configurations. | | `-jt or ` | Specify a ResourceManager. Applies only to job. | -| `-libjars ` | Specify comma separated jar files to include in the classpath. Applies only to job. | +| `-libjars ` | Specify comma separated jar files to include in the classpath. Applies only to job. | Hadoop Common Commands ====================== diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md index 0c5f4861c1afb..0de09250b582b 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md @@ -133,18 +133,21 @@ In order to indicate a particular provider type and location, the user must prov 1. The `UserProvider`, which is represented by the provider URI `user:///`, is used to retrieve credentials from a user's Credentials file. This file is used to store various tokens, secrets and passwords that are needed by executing jobs and applications. 2. The `JavaKeyStoreProvider`, which is represented by the provider URI `jceks://SCHEME/path-to-keystore`, is used to retrieve credentials from a Java keystore file in a filesystem `` The underlying use of the Hadoop filesystem API allows credentials to be stored on the local filesystem or within cluster stores. -3. The `LocalJavaKeyStoreProvider`, which is represented by the provider URI `localjceks://file/path-to-keystore`, is used to access credentials from a Java keystore that is must be stored on the local filesystem. This is needed for credentials that would result in a recursive dependency on accessing HDFS. Anytime that your credential is required to gain access to HDFS we can't depend on getting a credential out of HDFS to do so. +3. The `LocalJavaKeyStoreProvider`, which is represented by the provider URI `localjceks://file/path-to-keystore`, is used to access credentials from a Java keystore that must be stored on the local filesystem. This is needed for credentials that would result in a recursive dependency on accessing HDFS. Anytime that your credential is required to gain access to HDFS we can't depend on getting a credential out of HDFS to do so. +4. The `BouncyCastleFIPSKeyStoreProvider`, which is represented by the provider URI `bcfks://SCHEME/path-to-keystore`, is used to retrieve credentials from a Bouncy Castle FIPS keystore file in a file system `` + The underlying use of the Hadoop filesystem API allows credentials to be stored on the local filesystem or within cluster stores. +5. The `LocalBcouncyCastleFIPSKeyStoreProvider`, which is represented by the provider URI `localbcfks://file/path-to-keystore`, is used to access credentials from a Bouncy Castle FIPS keystore that must be stored on the local filesystem. This is needed for credentials that would result in a recursive dependency on accessing HDFS. Anytime that your credential is required to gain access to HDFS we can't depend on getting a credential out of HDFS to do so. When credentials are stored in a filesystem, the following rules apply: -* Credentials stored in local `localjceks://` files are loaded in the process reading in the configuration. +* Credentials stored in local `localjceks://` or `localbcfks://` files are loaded in the process reading in the configuration. For use in a YARN application, this means that they must be visible across the entire cluster, in the local filesystems of the hosts. -* Credentials stored with the `jceks://` provider can be stored in the cluster filesystem, +* Credentials stored with the `jceks://` or `bcfks://` provider can be stored in the cluster filesystem, and so visible across the cluster —but not in the filesystem which requires the specific credentials for their access. -To wrap filesystem URIs with a `jceks` URI follow these steps: +To wrap filesystem URIs with a `jceks` URI follow these steps. Bouncy Castle FIPS provider follows a similar step by replacing `jceks` with `bcfks` along with OS/JDK level FIPS provider configured. 1. Take a filesystem URI such as `hdfs://namenode:9001/users/alice/secrets.jceks` 1. Place `jceks://` in front of the URL: `jceks://hdfs://namenode:9001/users/alice/secrets.jceks` diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FairCallQueue.md b/hadoop-common-project/hadoop-common/src/site/markdown/FairCallQueue.md index 887d3053d263e..504a559e31495 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/FairCallQueue.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/FairCallQueue.md @@ -91,6 +91,10 @@ This is configurable via the **identity provider**, which defaults to the **User provider simply uses the username of the client submitting the request. However, a custom identity provider can be used to performing throttling based on other groupings, or using an external identity provider. +If particular users submit important requests and you don't want to limit them, you can set them up as the +**service-users**. They are always scheduled into the high-priority queue and won't be included in the priority +computation of normal user calls. + ### Cost-based Fair Call Queue Though the fair call queue itself does a good job of mitigating the impact from users who submit a very high _number_ @@ -138,6 +142,7 @@ omitted. | decay-scheduler.backoff.responsetime.enable | DecayRpcScheduler | Whether or not to enable the backoff by response time feature. | false | | decay-scheduler.backoff.responsetime.thresholds | DecayRpcScheduler | The response time thresholds, as time durations, for each priority queue. If the average response time for a queue is above this threshold, backoff will occur in lower priority queues. This should be a comma-separated list of length equal to the number of priority levels. | Threshold increases by 10s per level (e.g., for 4 levels: `10s,20s,30s,40s`) | | decay-scheduler.metrics.top.user.count | DecayRpcScheduler | The number of top (i.e., heaviest) users to emit metric information about. | 10 | +| decay-scheduler.service-users | DecayRpcScheduler | Service users will always be scheduled into the highest-priority queue and won't be included in the priority computation of normal user calls. They are specified as a comma-separated list. | | | weighted-cost.lockshared | WeightedTimeCostProvider | The weight multiplier to apply to the time spent in the processing phase which holds a shared (read) lock. | 10 | | weighted-cost.lockexclusive | WeightedTimeCostProvider | The weight multiplier to apply to the time spent in the processing phase which holds an exclusive (write) lock. | 100 | | weighted-cost.{handler,lockfree,response} | WeightedTimeCostProvider | The weight multiplier to apply to the time spent in the processing phases which do not involve holding a lock. See `org.apache.hadoop.ipc.ProcessingDetails.Timing` for more details on each phase. | 1 | diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md index 9c5586913d900..c5080dba29e47 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md @@ -154,7 +154,7 @@ The -e option shows the erasure coding policy for each file. The output columns with -count -e are: DIR\_COUNT, FILE\_COUNT, CONTENT_SIZE, ERASURECODING\_POLICY, PATHNAME -The ERASURECODING\_POLICY is name of the policy for the file. If a erasure coding policy is setted on that file, it will return name of the policy. If no erasure coding policy is setted, it will return \"Replicated\" which means it use replication storage strategy. +The ERASURECODING\_POLICY is name of the policy for the file. If an erasure coding policy is set on that file, it will return the name of the policy. Otherwise, it will return \"Replicated\" which means it uses the replication storage strategy. The -s option shows the snapshot counts for each directory. @@ -813,6 +813,18 @@ Example: * `hadoop fs -truncate 55 /user/hadoop/file1 /user/hadoop/file2` * `hadoop fs -truncate -w 127 hdfs://nn1.example.com/user/hadoop/file1` +concat +-------- + +Usage: `hadoop fs -concat ` + +Concatenate existing source files into the target file. Target file and source +files should be in the same directory. + +Example: + +* `hadoop fs -concat hdfs://cluster/user/hadoop/target-file hdfs://cluster/user/hadoop/file-0 hdfs://cluster/user/hadoop/file-1` + usage ----- @@ -1092,6 +1104,7 @@ actually fail. | `setfattr` | generally unsupported permissions model | | `setrep`| has no effect | | `truncate` | generally unsupported | +| `concat` | generally unsupported | Different object store clients *may* support these commands: do consult the documentation and test against the target store. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 43a3f331f0f30..18d326395f95d 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -548,8 +548,8 @@ RouterRPCMetrics shows the statistics of the Router component in Router-based fe |:---- |:---- | | `ProcessingOp` | Number of operations the Router processed internally | | `ProxyOp` | Number of operations the Router proxied to a Namenode | -| `ProxyOpFailureStandby` | Number of operations to fail to reach NN | -| `ProxyOpFailureCommunicate` | Number of operations to hit a standby NN | +| `ProxyOpFailureStandby` | Number of operations to hit a standby NN | +| `ProxyOpFailureCommunicate` | Number of operations to fail to reach NN | | `ProxyOpNotImplemented` | Number of operations not implemented | | `RouterFailureStateStore` | Number of failed requests due to State Store unavailable | | `RouterFailureReadOnly` | Number of failed requests due to read only mount point | diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/NativeLibraries.md.vm b/hadoop-common-project/hadoop-common/src/site/markdown/NativeLibraries.md.vm index e4f720cee8ce1..1e62e94394f91 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/NativeLibraries.md.vm +++ b/hadoop-common-project/hadoop-common/src/site/markdown/NativeLibraries.md.vm @@ -47,7 +47,7 @@ Components The native hadoop library includes various components: -* Compression Codecs (bzip2, lz4, snappy, zlib) +* Compression Codecs (bzip2, lz4, zlib) * Native IO utilities for [HDFS Short-Circuit Local Reads](../hadoop-hdfs/ShortCircuitLocalReads.html) and [Centralized Cache Management in HDFS](../hadoop-hdfs/CentralizedCacheManagement.html) * CRC32 checksum implementation @@ -117,7 +117,6 @@ NativeLibraryChecker is a tool to check whether native libraries are loaded corr Native library checking: hadoop: true /home/ozawa/hadoop/lib/native/libhadoop.so.1.0.0 zlib: true /lib/x86_64-linux-gnu/libz.so.1 - snappy: true /usr/lib/libsnappy.so.1 zstd: true /usr/lib/libzstd.so.1 lz4: true revision:99 bzip2: false diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md index 856861f29e3c2..523fa40dbc14f 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md @@ -203,6 +203,8 @@ Setting `dfs.encrypt.data.transfer.cipher.suites` to `AES/CTR/NoPadding` activat AES offers the greatest cryptographic strength and the best performance. At this time, 3DES and RC4 have been used more often in Hadoop clusters. +You can also set `dfs.encrypt.data.transfer.cipher.suites` to `SM4/CTR/NoPadding` to activates SM4 encryption. By default, this is unspecified. The SM4 key bit length can be configured by setting `dfs.encrypt.data.transfer.cipher.key.bitlength` to 128, 192 or 256. The default is 128. + ### Data Encryption on HTTP Data transfer between Web-console and clients are protected by using SSL(HTTPS). SSL configuration is recommended but not required to configure Hadoop security with Kerberos. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md index 665e328447d5b..284a964f6e522 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md @@ -294,6 +294,24 @@ any optimizations. The atomicity and consistency constraints are as for `listStatus(Path, PathFilter)`. +### `RemoteIterator listStatusIterator(Path p)` + +Return an iterator enumerating the `FileStatus` entries under +a path. This is similar to `listStatus(Path)` except the fact that +rather than returning an entire list, an iterator is returned. +The result is exactly the same as `listStatus(Path)`, provided no other +caller updates the directory during the listing. Having said that, this does +not guarantee atomicity if other callers are adding/deleting the files +inside the directory while listing is being performed. Different filesystems +may provide a more efficient implementation, for example S3A does the +listing in pages and fetches the next pages asynchronously while a +page is getting processed. + +Note that now since the initial listing is async, bucket/path existence +exception may show up later during next() call. + +Callers should prefer using listStatusIterator over listStatus as it +is incremental in nature. ### `FileStatus[] listStatus(Path[] paths)` @@ -1107,7 +1125,7 @@ Rename includes the calculation of the destination path. If the destination exists and is a directory, the final destination of the rename becomes the destination + the filename of the source path. - let dest = if (isDir(FS, src) and d != src) : + let dest = if (isDir(FS, d) and d != src) : d + [filename(src)] else : d @@ -1186,10 +1204,10 @@ If `src` is a directory then all its children will then exist under `dest`, whil `src` and its descendants will no longer exist. The names of the paths under `dest` will match those under `src`, as will the contents: - if isDir(FS, src) isDir(FS, dest) and src != dest : + if isDir(FS, src) and isDir(FS, dest) and src != dest : FS' where: not exists(FS', src) - and dest in FS'.Directories] + and dest in FS'.Directories and forall c in descendants(FS, src) : not exists(FS', c)) and forall c in descendants(FS, src) where isDir(FS, c): diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index dd9f41a7a3527..30ed4a4193783 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -172,6 +172,8 @@ public void initializeMemberVariables() { "ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds"); xmlPropsToSkipCompare.add( "ipc.[port_number].decay-scheduler.metrics.top.user.count"); + xmlPropsToSkipCompare.add( + "ipc.[port_number].decay-scheduler.service-users"); xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.lockshared"); xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.lockexclusive"); xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.handler"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java index cf42219f6d8b5..eae9a1fc744c0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java @@ -38,7 +38,7 @@ import org.w3c.dom.NodeList; import org.xml.sax.InputSource; -import com.google.common.base.Strings; +import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import org.apache.hadoop.http.HttpServer2; import org.junit.BeforeClass; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 81c53959478b4..a2273ef34faba 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -1062,6 +1062,38 @@ public void testRelativeIncludes() throws Exception { new File(new File(relConfig).getParent()).delete(); } + @Test + public void testRelativeIncludesWithLoadingViaUri() throws Exception { + tearDown(); + File configFile = new File("./tmp/test-config.xml"); + File configFile2 = new File("./tmp/test-config2.xml"); + + new File(configFile.getParent()).mkdirs(); + out = new BufferedWriter(new FileWriter(configFile2)); + startConfig(); + appendProperty("a", "b"); + endConfig(); + + out = new BufferedWriter(new FileWriter(configFile)); + startConfig(); + // Add the relative path instead of the absolute one. + startInclude(configFile2.getName()); + endInclude(); + appendProperty("c", "d"); + endConfig(); + + // verify that the includes file contains all properties + Path fileResource = new Path(configFile.toURI()); + conf.addResource(fileResource); + assertEquals("b", conf.get("a")); + assertEquals("d", conf.get("c")); + + // Cleanup + configFile.delete(); + configFile2.delete(); + new File(configFile.getParent()).delete(); + } + @Test public void testIntegerRanges() { Configuration conf = new Configuration(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java index efb813164af69..2c0d6025f2688 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java @@ -47,8 +47,8 @@ import org.junit.Before; import org.junit.Test; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles; public class TestConfigurationDeprecation { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationSubclass.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationSubclass.java index e15e699534d31..51d23d8038b0b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationSubclass.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationSubclass.java @@ -53,8 +53,9 @@ public void testReloadNotQuiet() throws Throwable { SubConf conf = new SubConf(true); conf.setQuietMode(false); assertFalse(conf.isReloaded()); + // adding a resource does not force a reload. conf.addResource("not-a-valid-resource"); - assertTrue(conf.isReloaded()); + assertFalse(conf.isReloaded()); try { Properties properties = conf.getProperties(); fail("Should not have got here"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestReconfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestReconfiguration.java index dd801a53a3dae..4948df9b1f4cb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestReconfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestReconfiguration.java @@ -19,7 +19,7 @@ package org.apache.hadoop.conf; import java.util.function.Supplier; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.conf.ReconfigurationUtil.PropertyChange; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java index eca23a7f2a372..c0fdc51b1389b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java @@ -17,6 +17,12 @@ */ package org.apache.hadoop.crypto; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic. + HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic. + HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic. + HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -43,7 +49,8 @@ import org.junit.Before; import org.junit.Test; -import com.google.common.primitives.Longs; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Longs; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,10 +64,14 @@ public class TestCryptoCodec { private Configuration conf = new Configuration(); private int count = 10000; private int seed = new Random().nextInt(); - private final String jceCodecClass = + private final String jceAesCodecClass = "org.apache.hadoop.crypto.JceAesCtrCryptoCodec"; - private final String opensslCodecClass = + private final String jceSm4CodecClass = + "org.apache.hadoop.crypto.JceSm4CtrCryptoCodec"; + private final String opensslAesCodecClass = "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec"; + private final String opensslSm4CodecClass = + "org.apache.hadoop.crypto.OpensslSm4CtrCryptoCodec"; @Before public void setUp() throws IOException { @@ -77,15 +88,49 @@ public void testJceAesCtrCryptoCodec() throws Exception { Assume.assumeTrue(false); } Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); - cryptoCodecTest(conf, seed, 0, jceCodecClass, jceCodecClass, iv); - cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass, iv); - cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass, iv); + cryptoCodecTest(conf, seed, 0, + jceAesCodecClass, jceAesCodecClass, iv); + cryptoCodecTest(conf, seed, count, + jceAesCodecClass, jceAesCodecClass, iv); + cryptoCodecTest(conf, seed, count, + jceAesCodecClass, opensslAesCodecClass, iv); // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff for(int i = 0; i < 8; i++) { iv[8 + i] = (byte) 0xff; } - cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass, iv); - cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass, iv); + cryptoCodecTest(conf, seed, count, + jceAesCodecClass, jceAesCodecClass, iv); + cryptoCodecTest(conf, seed, count, + jceAesCodecClass, opensslAesCodecClass, iv); + } + + @Test(timeout=120000) + public void testJceSm4CtrCryptoCodec() throws Exception { + GenericTestUtils.assumeInNativeProfile(); + if (!NativeCodeLoader.buildSupportsOpenssl()) { + LOG.warn("Skipping test since openSSL library not loaded"); + Assume.assumeTrue(false); + } + conf.set(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, "SM4/CTR/NoPadding"); + conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY, + JceSm4CtrCryptoCodec.class.getName()); + conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, + BouncyCastleProvider.PROVIDER_NAME); + Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); + cryptoCodecTest(conf, seed, 0, + jceSm4CodecClass, jceSm4CodecClass, iv); + cryptoCodecTest(conf, seed, count, + jceSm4CodecClass, jceSm4CodecClass, iv); + cryptoCodecTest(conf, seed, count, + jceSm4CodecClass, opensslSm4CodecClass, iv); + // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff + for(int i = 0; i < 8; i++) { + iv[8 + i] = (byte) 0xff; + } + cryptoCodecTest(conf, seed, count, + jceSm4CodecClass, jceSm4CodecClass, iv); + cryptoCodecTest(conf, seed, count, + jceSm4CodecClass, opensslSm4CodecClass, iv); } @Test(timeout=120000) @@ -96,15 +141,46 @@ public void testOpensslAesCtrCryptoCodec() throws Exception { Assume.assumeTrue(false); } Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); - cryptoCodecTest(conf, seed, 0, opensslCodecClass, opensslCodecClass, iv); - cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass, iv); - cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass, iv); + cryptoCodecTest(conf, seed, 0, + opensslAesCodecClass, opensslAesCodecClass, iv); + cryptoCodecTest(conf, seed, count, + opensslAesCodecClass, opensslAesCodecClass, iv); + cryptoCodecTest(conf, seed, count, + opensslAesCodecClass, jceAesCodecClass, iv); // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff for(int i = 0; i < 8; i++) { iv[8 + i] = (byte) 0xff; } - cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass, iv); - cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass, iv); + cryptoCodecTest(conf, seed, count, + opensslAesCodecClass, opensslAesCodecClass, iv); + cryptoCodecTest(conf, seed, count, + opensslAesCodecClass, jceAesCodecClass, iv); + } + + @Test(timeout=120000) + public void testOpensslSm4CtrCryptoCodec() throws Exception { + GenericTestUtils.assumeInNativeProfile(); + if (!NativeCodeLoader.buildSupportsOpenssl()) { + LOG.warn("Skipping test since openSSL library not loaded"); + Assume.assumeTrue(false); + } + conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, + BouncyCastleProvider.PROVIDER_NAME); + Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); + cryptoCodecTest(conf, seed, 0, + opensslSm4CodecClass, opensslSm4CodecClass, iv); + cryptoCodecTest(conf, seed, count, + opensslSm4CodecClass, opensslSm4CodecClass, iv); + cryptoCodecTest(conf, seed, count, + opensslSm4CodecClass, jceSm4CodecClass, iv); + // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff + for(int i = 0; i < 8; i++) { + iv[8 + i] = (byte) 0xff; + } + cryptoCodecTest(conf, seed, count, + opensslSm4CodecClass, opensslSm4CodecClass, iv); + cryptoCodecTest(conf, seed, count, + opensslSm4CodecClass, jceSm4CodecClass, iv); } private void cryptoCodecTest(Configuration conf, int seed, int count, diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithJceSm4CtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithJceSm4CtrCryptoCodec.java new file mode 100644 index 0000000000000..62573ede7d1ea --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithJceSm4CtrCryptoCodec.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto; + +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.junit.BeforeClass; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic. + HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY; +import static org.assertj.core.api.Assertions.assertThat; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY; + +public class TestCryptoStreamsWithJceSm4CtrCryptoCodec extends + TestCryptoStreams { + + @BeforeClass + public static void init() throws Exception { + Configuration conf = new Configuration(); + conf.set(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, "SM4/CTR/NoPadding"); + conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, + BouncyCastleProvider.PROVIDER_NAME); + conf.set( + CommonConfigurationKeysPublic. + HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY, + JceSm4CtrCryptoCodec.class.getName()); + codec = CryptoCodec.getInstance(conf); + assertThat(JceSm4CtrCryptoCodec.class.getCanonicalName()). + isEqualTo(codec.getClass().getCanonicalName()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java index 6b56aafe17c9c..74e1a0648ba97 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java @@ -21,14 +21,16 @@ import org.apache.hadoop.crypto.random.OsSecureRandom; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.test.Whitebox; import org.junit.BeforeClass; import org.junit.Test; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY; public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec extends TestCryptoStreams { @@ -51,6 +53,7 @@ public static void init() throws Exception { public void testCodecClosesRandom() throws Exception { GenericTestUtils.assumeInNativeProfile(); Configuration conf = new Configuration(); + conf.set(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, "AES/CTR/NoPadding"); conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY, OpensslAesCtrCryptoCodec.class.getName()); conf.set( @@ -61,13 +64,13 @@ public void testCodecClosesRandom() throws Exception { "Unable to instantiate codec " + OpensslAesCtrCryptoCodec.class .getName() + ", is the required " + "version of OpenSSL installed?", codecWithRandom); - OsSecureRandom random = - (OsSecureRandom) Whitebox.getInternalState(codecWithRandom, "random"); + OsSecureRandom random = (OsSecureRandom) + ((OpensslAesCtrCryptoCodec) codecWithRandom).getRandom(); // trigger the OsSecureRandom to create an internal FileInputStream random.nextBytes(new byte[10]); - assertNotNull(Whitebox.getInternalState(random, "stream")); + assertFalse(random.isClosed()); // verify closing the codec closes the codec's random's stream. codecWithRandom.close(); - assertNull(Whitebox.getInternalState(random, "stream")); + assertTrue(random.isClosed()); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslSm4CtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslSm4CtrCryptoCodec.java new file mode 100644 index 0000000000000..f6345557211f9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslSm4CtrCryptoCodec.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.random.OsSecureRandom; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic. + HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic. + HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY; + +public class TestCryptoStreamsWithOpensslSm4CtrCryptoCodec + extends TestCryptoStreams { + + @BeforeClass + public static void init() throws Exception { + GenericTestUtils.assumeInNativeProfile(); + Configuration conf = new Configuration(); + conf.set(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, "SM4/CTR/NoPadding"); + conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY, + OpensslSm4CtrCryptoCodec.class.getName()); + codec = CryptoCodec.getInstance(conf); + assertNotNull("Unable to instantiate codec " + + OpensslSm4CtrCryptoCodec.class.getName() + ", is the required " + + "version of OpenSSL installed?", codec); + assertEquals(OpensslSm4CtrCryptoCodec.class.getCanonicalName(), + codec.getClass().getCanonicalName()); + } + + @Test + public void testCodecClosesRandom() throws Exception { + GenericTestUtils.assumeInNativeProfile(); + Configuration conf = new Configuration(); + conf.set(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, "SM4/CTR/NoPadding"); + conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY, + OpensslSm4CtrCryptoCodec.class.getName()); + conf.set( + CommonConfigurationKeysPublic. + HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY, + OsSecureRandom.class.getName()); + CryptoCodec codecWithRandom = CryptoCodec.getInstance(conf); + assertNotNull("Unable to instantiate codec " + + OpensslSm4CtrCryptoCodec.class.getName() + ", is the required " + + "version of OpenSSL installed?", codecWithRandom); + OsSecureRandom random = (OsSecureRandom) + ((OpensslSm4CtrCryptoCodec) codecWithRandom).getRandom(); + // trigger the OsSecureRandom to create an internal FileInputStream + random.nextBytes(new byte[10]); + assertFalse(random.isClosed()); + // verify closing the codec closes the codec's random's stream. + codecWithRandom.close(); + assertTrue(random.isClosed()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java index 55a9280d6260a..9b8638faa4b22 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java @@ -32,7 +32,7 @@ import org.junit.Assert; import org.junit.Test; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; public class TestValueQueue { Logger LOG = LoggerFactory.getLogger(TestValueQueue.class); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java index 7804c73fd2afc..616c66b0748db 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java @@ -62,7 +62,7 @@ import org.junit.rules.Timeout; import org.mockito.Mockito; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; public class TestLoadBalancingKMSClientProvider { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java index de278dd37ef46..2aa5407c056af 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java @@ -38,7 +38,7 @@ import org.junit.Test; import java.util.function.Supplier; -import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java index 72ae296c957b5..e3c4ee05c8c8f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java @@ -23,6 +23,7 @@ import static org.hamcrest.CoreMatchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -34,6 +35,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.util.StringUtils; import org.junit.Before; import org.junit.BeforeClass; @@ -175,7 +177,20 @@ public void testCopyDirFromWindowsLocalPath() throws Exception { checkPut(dirPath, targetDir, true); } - + @Test + public void testCopyBetweenFsEqualPath() throws Exception { + Path testRoot = new Path(testRootDir, "testPutFile"); + lfs.delete(testRoot, true); + lfs.mkdirs(testRoot); + + Path filePath = new Path(testRoot, "sameSourceTarget"); + lfs.create(filePath).close(); + final FileStatus status = lfs.getFileStatus(filePath); + LambdaTestUtils.intercept(PathOperationException.class, () -> + FileUtil.copy(lfs, status, lfs, filePath, false, true, conf) + ); + } + private void checkPut(Path srcPath, Path targetDir, boolean useWindowsPath) throws Exception { lfs.delete(targetDir, true); @@ -683,4 +698,27 @@ public void testPutSrcFileNoPerm() lfs.setPermission(src, new FsPermission((short)0755)); } } + + @Test + public void testLazyPersistDirectOverwrite() throws Exception { + Path testRoot = new Path(testRootDir, "testLazyPersistDirectOverwrite"); + try { + lfs.delete(testRoot, true); + lfs.mkdirs(testRoot); + Path filePath = new Path(testRoot, new Path("srcFile")); + lfs.create(filePath).close(); + // Put with overwrite in direct mode. + String[] argv = + new String[] {"-put", "-f", "-l", "-d", filePath.toString(), + filePath.toString()}; + assertEquals(0, shell.run(argv)); + + // Put without overwrite in direct mode shouldn't be success. + argv = new String[] {"-put", "-l", "-d", filePath.toString(), + filePath.toString()}; + assertNotEquals(0, shell.run(argv)); + } finally { + lfs.delete(testRoot, true); + } + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java index 2e7cb5d6342b3..62e7990674d3b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java @@ -101,7 +101,7 @@ public void testTouch() throws Exception { { assertThat( "Expected successful touch on a non-existent file with -c option", - shellRun("-touch", "-c", newFileName), is(not(0))); + shellRun("-touch", "-c", newFileName), is(0)); assertThat(lfs.exists(newFile), is(false)); } @@ -140,7 +140,7 @@ public void testTouch() throws Exception { Date dateObj = parseTimestamp(strTime); assertThat( - "Expected successful touch with a specified modificatiom time", + "Expected successful touch with a specified modification time", shellRun("-touch", "-m", "-t", strTime, newFileName), is(0)); // Verify if modification time is recorded correctly (and access time // remains unchanged). @@ -179,6 +179,16 @@ public void testTouch() throws Exception { assertThat("Expected failed touch with a missing timestamp", shellRun("-touch", "-t", newFileName), is(not(0))); } + + // Verify -c option when file exists. + String strTime = formatTimestamp(System.currentTimeMillis()); + Date dateObj = parseTimestamp(strTime); + assertThat( + "Expected successful touch on a non-existent file with -c option", + shellRun("-touch", "-c", "-t", strTime, newFileName), is(0)); + FileStatus fileStatus = lfs.getFileStatus(newFile); + assertThat(fileStatus.getAccessTime(), is(dateObj.getTime())); + assertThat(fileStatus.getModificationTime(), is(dateObj.getTime())); } private String formatTimestamp(long timeInMillis) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index 517f6ce016544..1384bb6a17f38 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.fs; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem.Statistics; import org.apache.hadoop.fs.permission.FsPermission; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java index c0d79c145f781..18f88084c3f48 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java @@ -34,7 +34,7 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; -import com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows; import static org.apache.hadoop.test.PlatformAssumptions.assumeWindows; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java index f63314d39292e..c0d9733bbb9a7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java @@ -24,6 +24,8 @@ import java.util.List; import java.util.UUID; +import org.assertj.core.api.Assertions; + import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FilterFileSystem; @@ -148,6 +150,7 @@ public void testListLocatedStatusEmptyDirectory() throws IOException { public void testComplexDirActions() throws Throwable { TreeScanResults tree = createTestTree(); checkListStatusStatusComplexDir(tree); + checkListStatusIteratorComplexDir(tree); checkListLocatedStatusStatusComplexDir(tree); checkListFilesComplexDirNonRecursive(tree); checkListFilesComplexDirRecursive(tree); @@ -169,6 +172,34 @@ protected void checkListStatusStatusComplexDir(TreeScanResults tree) listing.assertSizeEquals("listStatus()", TREE_FILES, TREE_WIDTH, 0); } + /** + * Test {@link FileSystem#listStatusIterator(Path)} on a complex + * directory tree. + * @param tree directory tree to list. + * @throws Throwable + */ + protected void checkListStatusIteratorComplexDir(TreeScanResults tree) + throws Throwable { + describe("Expect listStatusIterator to list all entries in top dir only"); + + FileSystem fs = getFileSystem(); + TreeScanResults listing = new TreeScanResults( + fs.listStatusIterator(tree.getBasePath())); + listing.assertSizeEquals("listStatus()", TREE_FILES, TREE_WIDTH, 0); + + List resWithoutCheckingHasNext = + iteratorToListThroughNextCallsAlone(fs + .listStatusIterator(tree.getBasePath())); + + List resWithCheckingHasNext = iteratorToList(fs + .listStatusIterator(tree.getBasePath())); + Assertions.assertThat(resWithCheckingHasNext) + .describedAs("listStatusIterator() should return correct " + + "results even if hasNext() calls are not made.") + .hasSameElementsAs(resWithoutCheckingHasNext); + + } + /** * Test {@link FileSystem#listLocatedStatus(Path)} on a complex * directory tree. @@ -322,6 +353,45 @@ public void testListStatusFile() throws Throwable { verifyStatusArrayMatchesFile(f, getFileSystem().listStatus(f)); } + @Test + public void testListStatusIteratorFile() throws Throwable { + describe("test the listStatusIterator(path) on a file"); + Path f = touchf("listStItrFile"); + + List statusList = (List) iteratorToList( + getFileSystem().listStatusIterator(f)); + validateListingForFile(f, statusList, false); + + List statusList2 = + (List) iteratorToListThroughNextCallsAlone( + getFileSystem().listStatusIterator(f)); + validateListingForFile(f, statusList2, true); + } + + /** + * Validate listing result for an input path which is file. + * @param f file. + * @param statusList list status of a file. + * @param nextCallAlone whether the listing generated just using + * next() calls. + */ + private void validateListingForFile(Path f, + List statusList, + boolean nextCallAlone) { + String msg = String.format("size of file list returned using %s should " + + "be 1", nextCallAlone ? + "next() calls alone" : "hasNext() and next() calls"); + Assertions.assertThat(statusList) + .describedAs(msg) + .hasSize(1); + Assertions.assertThat(statusList.get(0).getPath()) + .describedAs("path returned should match with the input path") + .isEqualTo(f); + Assertions.assertThat(statusList.get(0).isFile()) + .describedAs("path returned should be a file") + .isEqualTo(true); + } + @Test public void testListFilesFile() throws Throwable { describe("test the listStatus(path) on a file"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java index 31926964c897c..d0b067ad6e6ae 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java @@ -28,7 +28,7 @@ import java.util.Random; import java.util.concurrent.CompletableFuture; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import org.assertj.core.api.Assertions; import org.junit.Assume; import org.junit.Test; @@ -807,12 +807,17 @@ public void testConcurrentUploads() throws Throwable { // now upload part 2. complete(file, upload2, partHandles2); - // and await the visible length to match - eventually(timeToBecomeConsistentMillis(), - () -> verifyFileLength(file, size2), - new LambdaTestUtils.ProportionalRetryInterval( - CONSISTENCY_INTERVAL, - timeToBecomeConsistentMillis())); + + // and await the visible length to match, if this FS is not + // consistent. + final int consistencyDelay = timeToBecomeConsistentMillis(); + if (consistencyDelay > 0) { + eventually(consistencyDelay, + () -> verifyFileLength(file, size2), + new LambdaTestUtils.ProportionalRetryInterval( + CONSISTENCY_INTERVAL, + consistencyDelay)); + } verifyContents(file, digest2, size2); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRenameTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRenameTest.java index 78ff2541483a3..e032604b5788c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRenameTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRenameTest.java @@ -104,29 +104,43 @@ public void testRenameFileOverExistingFile() throws Throwable { assertIsFile(destFile); boolean renameOverwritesDest = isSupported(RENAME_OVERWRITES_DEST); boolean renameReturnsFalseOnRenameDestExists = - !isSupported(RENAME_RETURNS_FALSE_IF_DEST_EXISTS); + isSupported(RENAME_RETURNS_FALSE_IF_DEST_EXISTS); + assertFalse(RENAME_OVERWRITES_DEST + " and " + + RENAME_RETURNS_FALSE_IF_DEST_EXISTS + " cannot be both supported", + renameOverwritesDest && renameReturnsFalseOnRenameDestExists); + String expectedTo = "expected rename(" + srcFile + ", " + destFile + ") to "; + boolean destUnchanged = true; try { + // rename is rejected by returning 'false' or throwing an exception boolean renamed = rename(srcFile, destFile); + destUnchanged = !renamed; if (renameOverwritesDest) { - // the filesystem supports rename(file, file2) by overwriting file2 - - assertTrue("Rename returned false", renamed); - destUnchanged = false; + assertTrue(expectedTo + "overwrite destination, but got false", + renamed); + } else if (renameReturnsFalseOnRenameDestExists) { + assertFalse(expectedTo + "be rejected with false, but destination " + + "was overwritten", renamed); + } else if (renamed) { + String destDirLS = generateAndLogErrorListing(srcFile, destFile); + getLogger().error("dest dir {}", destDirLS); + + fail(expectedTo + "be rejected with exception, but got overwritten"); } else { - // rename is rejected by returning 'false' or throwing an exception - if (renamed && !renameReturnsFalseOnRenameDestExists) { - //expected an exception - String destDirLS = generateAndLogErrorListing(srcFile, destFile); - getLogger().error("dest dir {}", destDirLS); - fail("expected rename(" + srcFile + ", " + destFile + " ) to fail," + - " but got success and destination of " + destDirLS); - } + fail(expectedTo + "be rejected with exception, but got false"); } } catch (FileAlreadyExistsException e) { + // rename(file, file2) should throw exception iff + // it neither overwrites nor returns false + assertFalse(expectedTo + "overwrite destination, but got exception", + renameOverwritesDest); + assertFalse(expectedTo + "be rejected with false, but got exception", + renameReturnsFalseOnRenameDestExists); + handleExpectedException(e); } + // verify that the destination file is as expected based on the expected // outcome verifyFileContents(getFileSystem(), destFile, diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java index 27c6933ae1885..6eaa56bab73f7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java @@ -22,13 +22,16 @@ import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.junit.Test; +import org.assertj.core.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.RemoteIterator; @@ -39,6 +42,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; import static org.apache.hadoop.fs.contract.ContractTestUtils.deleteChildren; import static org.apache.hadoop.fs.contract.ContractTestUtils.dumpStats; +import static org.apache.hadoop.fs.contract.ContractTestUtils.iteratorToList; import static org.apache.hadoop.fs.contract.ContractTestUtils.listChildren; import static org.apache.hadoop.fs.contract.ContractTestUtils.toList; import static org.apache.hadoop.fs.contract.ContractTestUtils.treeWalk; @@ -242,6 +246,13 @@ public void testSimpleRootListing() throws IOException { + "listStatus = " + listStatusResult + "listFiles = " + listFilesResult, fileList.size() <= statuses.length); + List statusList = (List) iteratorToList( + fs.listStatusIterator(root)); + Assertions.assertThat(statusList) + .describedAs("Result of listStatus(/) and listStatusIterator(/)" + + " must match") + .hasSameElementsAs(Arrays.stream(statuses) + .collect(Collectors.toList())); } @Test diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java index ca8e4a053beac..d34178489c81d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java @@ -317,7 +317,7 @@ public void testPositionedBulkReadDoesntChangePosition() throws Throwable { int v = 256; byte[] readBuffer = new byte[v]; - assertEquals(v, instream.read(128, readBuffer, 0, v)); + instream.readFully(128, readBuffer, 0, v); //have gone back assertEquals(40000, instream.getPos()); //content is the same too @@ -572,8 +572,7 @@ public void testReadSmallFile() throws Throwable { // now read the entire file in one go byte[] fullFile = new byte[TEST_FILE_LEN]; - assertEquals(TEST_FILE_LEN, - instream.read(0, fullFile, 0, fullFile.length)); + instream.readFully(0, fullFile, 0, fullFile.length); assertEquals(0, instream.getPos()); // now read past the end of the file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractUnbufferTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractUnbufferTest.java index 5eb1e892f83d5..adaf0a910c620 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractUnbufferTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractUnbufferTest.java @@ -137,7 +137,8 @@ protected void validateFileContents(FSDataInputStream stream, int length, throws IOException { byte[] streamData = new byte[length]; assertEquals("failed to read expected number of bytes from " - + "stream", length, stream.read(streamData)); + + "stream. This may be transient", + length, stream.read(streamData)); byte[] validateFileBytes; if (startIndex == 0 && length == fileBytes.length) { validateFileBytes = fileBytes; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java index 4789630f95f1c..39a41d01c458a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java @@ -1453,6 +1453,52 @@ public static List toList( return list; } + /** + * Convert a remote iterator over file status results into a list. + * The utility equivalents in commons collection and guava cannot be + * used here, as this is a different interface, one whose operators + * can throw IOEs. + * @param iterator input iterator + * @return the file status entries as a list. + * @throws IOException + */ + public static List iteratorToList( + RemoteIterator iterator) throws IOException { + List list = new ArrayList<>(); + while (iterator.hasNext()) { + list.add(iterator.next()); + } + return list; + } + + + /** + * Convert a remote iterator over file status results into a list. + * This uses {@link RemoteIterator#next()} calls only, expecting + * a raised {@link NoSuchElementException} exception to indicate that + * the end of the listing has been reached. This iteration strategy is + * designed to verify that the implementation of the remote iterator + * generates results and terminates consistently with the {@code hasNext/next} + * iteration. More succinctly "verifies that the {@code next()} operator + * isn't relying on {@code hasNext()} to always be called during an iteration. + * @param iterator input iterator + * @return the status entries as a list. + * @throws IOException IO problems + */ + @SuppressWarnings("InfiniteLoopStatement") + public static List iteratorToListThroughNextCallsAlone( + RemoteIterator iterator) throws IOException { + List list = new ArrayList<>(); + try { + while (true) { + list.add(iterator.next()); + } + } catch (NoSuchElementException expected) { + // ignored + } + return list; + } + /** * Convert a remote iterator over file status results into a list. * This uses {@link RemoteIterator#next()} calls only, expecting @@ -1602,7 +1648,7 @@ public TreeScanResults(Path basePath) { * @param results results of the listFiles/listStatus call. * @throws IOException IO problems during the iteration. */ - public TreeScanResults(RemoteIterator results) + public TreeScanResults(RemoteIterator results) throws IOException { while (results.hasNext()) { add(results.next()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/ftp/TestFTPFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/ftp/TestFTPFileSystem.java index d3750e64469b2..8155d8e2b2ba1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/ftp/TestFTPFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/ftp/TestFTPFileSystem.java @@ -23,7 +23,7 @@ import java.nio.file.Files; import java.util.Comparator; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.net.ftp.FTP; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPFile; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java index 4b3bd2f94075c..1ccc3400788d1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java @@ -47,7 +47,7 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java index f101fed26bbf8..618cb0190a99a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java @@ -283,6 +283,7 @@ public void processPathWithQuotasByStorageTypesHeader() throws Exception { count.processOptions(options); String withStorageTypeHeader = // <----13---> <-------17------> <----13-----> <------17-------> + " NVDIMM_QUOTA REM_NVDIMM_QUOTA " + " SSD_QUOTA REM_SSD_QUOTA DISK_QUOTA REM_DISK_QUOTA " + // <----13---> <-------17------> "ARCHIVE_QUOTA REM_ARCHIVE_QUOTA PROVIDED_QUOTA REM_PROVIDED_QUOTA " + @@ -337,6 +338,7 @@ public void processPathWithQuotasByQTVH() throws Exception { count.processOptions(options); String withStorageTypeHeader = // <----13---> <-------17------> + " NVDIMM_QUOTA REM_NVDIMM_QUOTA " + " SSD_QUOTA REM_SSD_QUOTA " + " DISK_QUOTA REM_DISK_QUOTA " + "ARCHIVE_QUOTA REM_ARCHIVE_QUOTA " + @@ -495,7 +497,7 @@ public void getDescription() { + "Otherwise, it displays the quota and usage for all the storage \n" + "types that support quota. The list of possible storage " + "types(case insensitive):\n" - + "ram_disk, ssd, disk and archive.\n" + + "ram_disk, ssd, disk, archive and nvdimm.\n" + "It can also pass the value '', 'all' or 'ALL' to specify all the " + "storage types.\n" + "The -u option shows the quota and \n" diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestFsShellConcat.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestFsShellConcat.java new file mode 100644 index 0000000000000..a2c4d3a1972c7 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestFsShellConcat.java @@ -0,0 +1,167 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.shell; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.net.URI; +import java.util.Random; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.assertj.core.api.Assertions; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsShell; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.mockito.ArgumentMatchers.any; +import static org.junit.Assert.assertEquals; + +/** + * Test Concat. + */ +public class TestFsShellConcat extends AbstractHadoopTestBase { + + private static Configuration conf; + private static FsShell shell; + private static LocalFileSystem lfs; + private static Path testRootDir; + private static Path dstPath; + + @Before + public void before() throws IOException { + conf = new Configuration(); + shell = new FsShell(conf); + lfs = FileSystem.getLocal(conf); + testRootDir = lfs.makeQualified(new Path(GenericTestUtils.getTempPath( + "testFsShellCopy"))); + + lfs.delete(testRootDir, true); + lfs.mkdirs(testRootDir); + lfs.setWorkingDirectory(testRootDir); + dstPath = new Path(testRootDir, "dstFile"); + lfs.create(dstPath).close(); + + Random random = new Random(); + for (int i = 0; i < 10; i++) { + OutputStream out = + lfs.create(new Path(testRootDir, String.format("file-%02d", i))); + out.write(random.nextInt()); + out.close(); + } + } + + @Test + public void testConcat() throws Exception { + // Read concatenated files to build the expected file content. + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (int i = 0; i < 10; i++) { + try (InputStream in = lfs + .open(new Path(testRootDir, String.format("file-%02d", i)))) { + IOUtils.copyBytes(in, out, 1024); + } + } + byte[] expectContent = out.toByteArray(); + + // Do concat. + FileSystem mockFs = Mockito.mock(FileSystem.class); + Mockito.doAnswer(invocation -> { + Object[] args = invocation.getArguments(); + Path target = (Path)args[0]; + Path[] src = (Path[]) args[1]; + mockConcat(target, src); + return null; + }).when(mockFs).concat(any(Path.class), any(Path[].class)); + Concat.setTestFs(mockFs); + shellRun(0, "-concat", dstPath.toString(), testRootDir+"/file-*"); + + // Verify concat result. + ContractTestUtils + .assertPathExists(lfs, "The target file doesn't exist.", dstPath); + Assertions.assertThat(lfs.listStatus(testRootDir).length).isEqualTo(1); + assertEquals(expectContent.length, lfs.getFileStatus(dstPath).getLen()); + out = new ByteArrayOutputStream(); + try (InputStream in = lfs.open(dstPath)) { + IOUtils.copyBytes(in, out, 1024); + } + // Verify content. + byte[] concatedContent = out.toByteArray(); + assertEquals(expectContent.length, concatedContent.length); + ContractTestUtils.compareByteArrays(expectContent, concatedContent, + expectContent.length); + } + + @Test + public void testUnsupportedFs() throws Exception { + FileSystem mockFs = Mockito.mock(FileSystem.class); + Mockito.doThrow( + new UnsupportedOperationException("Mock unsupported exception.")) + .when(mockFs).concat(any(Path.class), any(Path[].class)); + Mockito.doAnswer(invocationOnMock -> new URI("mockfs:///")).when(mockFs) + .getUri(); + Concat.setTestFs(mockFs); + final ByteArrayOutputStream err = new ByteArrayOutputStream(); + PrintStream oldErr = System.err; + System.setErr(new PrintStream(err)); + try { + shellRun(1, "-concat", dstPath.toString(), testRootDir + "/file-*"); + } finally { + System.setErr(oldErr); + } + System.err.print(err.toString()); + String expectedErrMsg = "Dest filesystem 'mockfs' doesn't support concat"; + Assertions.assertThat(err.toString().contains(expectedErrMsg)) + .withFailMessage("The err message should contain \"" + expectedErrMsg + + "\" message.").isTrue(); + } + + private void shellRun(int n, String... args) { + assertEquals(n, shell.run(args)); + } + + /** + * Simple simulation of concat. + */ + private void mockConcat(Path target, Path[] srcArray) throws IOException { + Path tmp = new Path(target.getParent(), target.getName() + ".bak"); + lfs.rename(target, tmp); + try (OutputStream out = lfs.create(target)) { + try (InputStream in = lfs.open(tmp)) { + IOUtils.copyBytes(in, out, 1024); + } + lfs.delete(tmp, true); + for (int i = 0; i < srcArray.length; i++) { + try (InputStream iin = lfs.open(srcArray[i])) { + IOUtils.copyBytes(iin, out, 1024); + } + lfs.delete(srcArray[i], true); + } + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestRegexMountPoint.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestRegexMountPoint.java new file mode 100644 index 0000000000000..5513b6005b41e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestRegexMountPoint.java @@ -0,0 +1,160 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.viewfs; + +import java.io.IOException; +import java.net.URI; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test Regex Mount Point. + */ +public class TestRegexMountPoint { + private static final Logger LOGGER = + LoggerFactory.getLogger(TestRegexMountPoint.class.getName()); + + private InodeTree inodeTree; + private Configuration conf; + + class TestRegexMountPointFileSystem { + public URI getUri() { + return uri; + } + + private URI uri; + + TestRegexMountPointFileSystem(URI uri) { + String uriStr = uri == null ? "null" : uri.toString(); + LOGGER.info("Create TestRegexMountPointFileSystem Via URI:" + uriStr); + this.uri = uri; + } + } + + @Before + public void setUp() throws Exception { + conf = new Configuration(); + ConfigUtil.addLink(conf, TestRegexMountPoint.class.getName(), "/mnt", + URI.create("file:///")); + + inodeTree = new InodeTree(conf, + TestRegexMountPoint.class.getName(), null, false) { + @Override + protected TestRegexMountPointFileSystem getTargetFileSystem( + final URI uri) { + return new TestRegexMountPointFileSystem(uri); + } + + @Override + protected TestRegexMountPointFileSystem getTargetFileSystem( + final INodeDir dir) { + return new TestRegexMountPointFileSystem(null); + } + + @Override + protected TestRegexMountPointFileSystem getTargetFileSystem( + final String settings, final URI[] mergeFsURIList) { + return new TestRegexMountPointFileSystem(null); + } + }; + } + + @After + public void tearDown() throws Exception { + inodeTree = null; + } + + @Test + public void testGetVarListInString() throws IOException { + String srcRegex = "/(\\w+)"; + String target = "/$0/${1}/$1/${2}/${2}"; + RegexMountPoint regexMountPoint = + new RegexMountPoint(inodeTree, srcRegex, target, null); + regexMountPoint.initialize(); + Map> varMap = regexMountPoint.getVarInDestPathMap(); + Assert.assertEquals(varMap.size(), 3); + Assert.assertEquals(varMap.get("0").size(), 1); + Assert.assertTrue(varMap.get("0").contains("$0")); + Assert.assertEquals(varMap.get("1").size(), 2); + Assert.assertTrue(varMap.get("1").contains("${1}")); + Assert.assertTrue(varMap.get("1").contains("$1")); + Assert.assertEquals(varMap.get("2").size(), 1); + Assert.assertTrue(varMap.get("2").contains("${2}")); + } + + @Test + public void testResolve() throws IOException { + String regexStr = "^/user/(?\\w+)"; + String dstPathStr = "/namenode1/testResolve/$username"; + String settingsStr = null; + RegexMountPoint regexMountPoint = + new RegexMountPoint(inodeTree, regexStr, dstPathStr, settingsStr); + regexMountPoint.initialize(); + InodeTree.ResolveResult resolveResult = + regexMountPoint.resolve("/user/hadoop/file1", true); + Assert.assertEquals(resolveResult.kind, InodeTree.ResultKind.EXTERNAL_DIR); + Assert.assertTrue( + resolveResult.targetFileSystem + instanceof TestRegexMountPointFileSystem); + Assert.assertEquals("/user/hadoop", resolveResult.resolvedPath); + Assert.assertTrue( + resolveResult.targetFileSystem + instanceof TestRegexMountPointFileSystem); + Assert.assertEquals("/namenode1/testResolve/hadoop", + ((TestRegexMountPointFileSystem) resolveResult.targetFileSystem) + .getUri().toString()); + Assert.assertEquals("/file1", resolveResult.remainingPath.toString()); + } + + @Test + public void testResolveWithInterceptor() throws IOException { + String regexStr = "^/user/(?\\w+)"; + String dstPathStr = "/namenode1/testResolve/$username"; + // Replace "_" with "-" + RegexMountPointResolvedDstPathReplaceInterceptor interceptor = + new RegexMountPointResolvedDstPathReplaceInterceptor("_", "-"); + // replaceresolvedpath:_:- + String settingsStr = interceptor.serializeToString(); + RegexMountPoint regexMountPoint = + new RegexMountPoint(inodeTree, regexStr, dstPathStr, settingsStr); + regexMountPoint.initialize(); + InodeTree.ResolveResult resolveResult = + regexMountPoint.resolve("/user/hadoop_user1/file_index", true); + Assert.assertEquals(resolveResult.kind, InodeTree.ResultKind.EXTERNAL_DIR); + Assert.assertTrue( + resolveResult.targetFileSystem + instanceof TestRegexMountPointFileSystem); + Assert.assertEquals("/user/hadoop_user1", resolveResult.resolvedPath); + Assert.assertTrue( + resolveResult.targetFileSystem + instanceof TestRegexMountPointFileSystem); + Assert.assertEquals("/namenode1/testResolve/hadoop-user1", + ((TestRegexMountPointFileSystem) resolveResult.targetFileSystem) + .getUri().toString()); + Assert.assertEquals("/file_index", + resolveResult.remainingPath.toString()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestRegexMountPointInterceptorFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestRegexMountPointInterceptorFactory.java new file mode 100644 index 0000000000000..c567944ffe307 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestRegexMountPointInterceptorFactory.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.viewfs; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Test Regex Mount Point Interceptor Factory. + */ +public class TestRegexMountPointInterceptorFactory { + + @Test + public void testCreateNormalCase() { + String replaceInterceptorStr = + RegexMountPointInterceptorType.REPLACE_RESOLVED_DST_PATH.getConfigName() + + Character.toString(RegexMountPoint.INTERCEPTOR_INTERNAL_SEP) + + "src" + Character + .toString(RegexMountPoint.INTERCEPTOR_INTERNAL_SEP) + "replace"; + RegexMountPointInterceptor interceptor = + RegexMountPointInterceptorFactory.create(replaceInterceptorStr); + Assert.assertTrue( + interceptor + instanceof RegexMountPointResolvedDstPathReplaceInterceptor); + } + + @Test + public void testCreateBadCase() { + String replaceInterceptorStr = + RegexMountPointInterceptorType.REPLACE_RESOLVED_DST_PATH.getConfigName() + + "___" + Character + .toString(RegexMountPoint.INTERCEPTOR_INTERNAL_SEP) + "src" + + Character.toString(RegexMountPoint.INTERCEPTOR_INTERNAL_SEP) + + "replace"; + RegexMountPointInterceptor interceptor = + RegexMountPointInterceptorFactory.create(replaceInterceptorStr); + Assert.assertTrue(interceptor == null); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestRegexMountPointResolvedDstPathReplaceInterceptor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestRegexMountPointResolvedDstPathReplaceInterceptor.java new file mode 100644 index 0000000000000..9fdf0f6ac9c5c --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestRegexMountPointResolvedDstPathReplaceInterceptor.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.viewfs; + +import java.io.IOException; + +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.hadoop.fs.viewfs.RegexMountPointInterceptorType.REPLACE_RESOLVED_DST_PATH; + +/** + * Test RegexMountPointResolvedDstPathReplaceInterceptor. + */ +public class TestRegexMountPointResolvedDstPathReplaceInterceptor { + + public String createSerializedString(String regex, String replaceString) { + return REPLACE_RESOLVED_DST_PATH.getConfigName() + + RegexMountPoint.INTERCEPTOR_INTERNAL_SEP + regex + + RegexMountPoint.INTERCEPTOR_INTERNAL_SEP + replaceString; + } + + @Test + public void testDeserializeFromStringNormalCase() throws IOException { + String srcRegex = "-"; + String replaceString = "_"; + String serializedString = createSerializedString(srcRegex, replaceString); + RegexMountPointResolvedDstPathReplaceInterceptor interceptor = + RegexMountPointResolvedDstPathReplaceInterceptor + .deserializeFromString(serializedString); + Assert.assertEquals(srcRegex, interceptor.getSrcRegexString()); + Assert.assertEquals(replaceString, interceptor.getReplaceString()); + Assert.assertNull(interceptor.getSrcRegexPattern()); + interceptor.initialize(); + Assert.assertEquals(srcRegex, + interceptor.getSrcRegexPattern().toString()); + } + + @Test + public void testDeserializeFromStringBadCase() throws IOException { + String srcRegex = "-"; + String replaceString = "_"; + String serializedString = createSerializedString(srcRegex, replaceString); + serializedString = serializedString + ":ddd"; + RegexMountPointResolvedDstPathReplaceInterceptor interceptor = + RegexMountPointResolvedDstPathReplaceInterceptor + .deserializeFromString(serializedString); + Assert.assertNull(interceptor); + } + + @Test + public void testSerialization() { + String srcRegex = "word1"; + String replaceString = "word2"; + String serializedString = createSerializedString(srcRegex, replaceString); + RegexMountPointResolvedDstPathReplaceInterceptor interceptor = + new RegexMountPointResolvedDstPathReplaceInterceptor(srcRegex, + replaceString); + Assert.assertEquals(interceptor.serializeToString(), serializedString); + } + + @Test + public void testInterceptSource() { + String srcRegex = "word1"; + String replaceString = "word2"; + RegexMountPointResolvedDstPathReplaceInterceptor interceptor = + new RegexMountPointResolvedDstPathReplaceInterceptor(srcRegex, + replaceString); + String sourcePath = "/a/b/l3/dd"; + Assert.assertEquals(sourcePath, interceptor.interceptSource(sourcePath)); + } + + @Test + public void testInterceptResolve() throws IOException { + String pathAfterResolution = "/user-hadoop"; + + String srcRegex = "hadoop"; + String replaceString = "hdfs"; + RegexMountPointResolvedDstPathReplaceInterceptor interceptor = + new RegexMountPointResolvedDstPathReplaceInterceptor(srcRegex, + replaceString); + interceptor.initialize(); + Assert.assertEquals("/user-hdfs", + interceptor.interceptResolvedDestPathStr(pathAfterResolution)); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsOverloadSchemeListStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsOverloadSchemeListStatus.java index 300fdd8b333f1..7afc78981f6e3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsOverloadSchemeListStatus.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsOverloadSchemeListStatus.java @@ -127,19 +127,30 @@ public void testListStatusACL() throws IOException, URISyntaxException { /** * Tests that ViewFSOverloadScheme should consider initialized fs as fallback - * if there are no mount links configured. + * if there are no mount links configured. It should add fallback with the + * chrootedFS at it's uri's root. */ @Test(timeout = 30000) public void testViewFSOverloadSchemeWithoutAnyMountLinks() throws Exception { - try (FileSystem fs = FileSystem.get(TEST_DIR.toPath().toUri(), conf)) { + Path initUri = new Path(TEST_DIR.toURI().toString(), "init"); + try (FileSystem fs = FileSystem.get(initUri.toUri(), conf)) { ViewFileSystemOverloadScheme vfs = (ViewFileSystemOverloadScheme) fs; assertEquals(0, vfs.getMountPoints().length); - Path testFallBack = new Path("test", FILE_NAME); - assertTrue(vfs.mkdirs(testFallBack)); - FileStatus[] status = vfs.listStatus(testFallBack.getParent()); - assertEquals(FILE_NAME, status[0].getPath().getName()); - assertEquals(testFallBack.getName(), - vfs.getFileLinkStatus(testFallBack).getPath().getName()); + Path testOnFallbackPath = new Path(TEST_DIR.toURI().toString(), "test"); + assertTrue(vfs.mkdirs(testOnFallbackPath)); + FileStatus[] status = vfs.listStatus(testOnFallbackPath.getParent()); + assertEquals(Path.getPathWithoutSchemeAndAuthority(testOnFallbackPath), + Path.getPathWithoutSchemeAndAuthority(status[0].getPath())); + //Check directly on localFS. The fallBackFs(localFS) should be chrooted + //at it's root. So, after + FileSystem lfs = vfs.getRawFileSystem(testOnFallbackPath, conf); + FileStatus[] statusOnLocalFS = + lfs.listStatus(testOnFallbackPath.getParent()); + assertEquals(testOnFallbackPath.getName(), + statusOnLocalFS[0].getPath().getName()); + //initUri should not have exist in lfs, as it would have chrooted on it's + // root only. + assertFalse(lfs.exists(initUri)); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java index 75557456edc3d..8ac447eb02e9b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java @@ -165,8 +165,8 @@ public void testGetFileChecksum() throws IOException { final Path path = new Path("/tmp/someFile"); FileSystem mockFS = Mockito.mock(FileSystem.class); InodeTree.ResolveResult res = - new InodeTree.ResolveResult(null, mockFS , null, - new Path("someFile")); + new InodeTree.ResolveResult(null, mockFS, null, + new Path("someFile"), true); @SuppressWarnings("unchecked") InodeTree fsState = Mockito.mock(InodeTree.class); Mockito.when(fsState.resolve(path.toString(), true)).thenReturn(res); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java index 21b0c159e2aae..73f6265eee72c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java @@ -547,8 +547,8 @@ public void testGetFileChecksum() throws AccessControlException, UnresolvedLinkException, IOException, URISyntaxException { AbstractFileSystem mockAFS = mock(AbstractFileSystem.class); InodeTree.ResolveResult res = - new InodeTree.ResolveResult(null, mockAFS , null, - new Path("someFile")); + new InodeTree.ResolveResult(null, mockAFS, null, + new Path("someFile"), true); @SuppressWarnings("unchecked") InodeTree fsState = mock(InodeTree.class); when(fsState.resolve(anyString(), anyBoolean())).thenReturn(res); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java index be6181157c0dd..666000b2f48ae 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java @@ -51,7 +51,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Copy-paste of ClientBase from ZooKeeper, but without any of the diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java index 0c2530739fa49..b5739f7935ed7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java @@ -35,7 +35,7 @@ import org.apache.hadoop.security.AccessControlException; import org.mockito.Mockito; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/MiniZKFCCluster.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/MiniZKFCCluster.java index f63d267f291c0..3c9713bf5fa1d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/MiniZKFCCluster.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/MiniZKFCCluster.java @@ -37,8 +37,8 @@ import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.server.ZooKeeperServer; -import com.google.common.base.Preconditions; -import com.google.common.primitives.Ints; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java index bbef9ef65b326..badd5afc5e91b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java @@ -37,7 +37,7 @@ import org.mockito.AdditionalMatchers; import org.mockito.Mockito; -import com.google.common.primitives.Ints; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; import org.slf4j.event.Level; /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAAdmin.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAAdmin.java index 63b9c63646d8b..13f7eccd55aea 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAAdmin.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAAdmin.java @@ -30,8 +30,8 @@ import org.junit.Before; import org.junit.Test; -import com.google.common.base.Charsets; -import com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java index b929bcb2490b0..972113eefa91f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java @@ -29,7 +29,7 @@ import org.junit.Test; import org.mockito.Mockito; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; public class TestNodeFencer { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java index fc36b1dd846a4..dcff9e30cdba2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java @@ -23,7 +23,7 @@ import java.net.InetSocketAddress; import java.util.List; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.util.Shell; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java index 54df39955d6cf..a72f06f748729 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java @@ -23,8 +23,8 @@ import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.util.Random; -import com.google.common.base.Charsets; -import com.google.common.primitives.Bytes; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Bytes; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -268,6 +268,8 @@ public void testClear() throws Exception { 0, text.getBytes().length); assertEquals("String's length must be zero", 0, text.getLength()); + assertEquals("String's text length must be zero", + 0, text.getTextLength()); // Test if clear works as intended text = new Text("abcd\u20acbdcd\u20ac"); @@ -280,6 +282,8 @@ public void testClear() throws Exception { text.getBytes().length >= len); assertEquals("Length of the string must be reset to 0 after clear()", 0, text.getLength()); + assertEquals("Text length of the string must be reset to 0 after clear()", + 0, text.getTextLength()); } @Test @@ -288,9 +292,12 @@ public void testTextText() throws CharacterCodingException { Text b=new Text("a"); b.set(a); assertEquals("abc", b.toString()); + assertEquals(3, a.getTextLength()); + assertEquals(3, b.getTextLength()); a.append("xdefgxxx".getBytes(), 1, 4); assertEquals("modified aliased string", "abc", b.toString()); assertEquals("appended string incorrectly", "abcdefg", a.toString()); + assertEquals("This should reflect in the lenght", 7, a.getTextLength()); // add an extra byte so that capacity = 10 and length = 8 a.append(new byte[]{'d'}, 0, 1); assertEquals(10, a.getBytes().length); @@ -392,16 +399,19 @@ public void testReadWithKnownLength() throws IOException { in.reset(inputBytes, inputBytes.length); text.readWithKnownLength(in, 5); assertEquals("hello", text.toString()); + assertEquals(5, text.getTextLength()); // Read longer length, make sure it lengthens in.reset(inputBytes, inputBytes.length); text.readWithKnownLength(in, 7); assertEquals("hello w", text.toString()); + assertEquals(7, text.getTextLength()); // Read shorter length, make sure it shortens in.reset(inputBytes, inputBytes.length); text.readWithKnownLength(in, 2); assertEquals("he", text.toString()); + assertEquals(2, text.getTextLength()); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java index 8be2dce06d1fe..ec42e4625e046 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java @@ -39,10 +39,10 @@ import org.apache.log4j.Logger; import org.junit.Assert; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; import static org.junit.Assert.*; public class CompressDecompressTester { @@ -79,27 +79,6 @@ public ImmutableList> filterOnAssumeWhat( }; } - private static boolean isNativeSnappyLoadable() { - boolean snappyAvailable = false; - boolean loaded = false; - try { - System.loadLibrary("snappy"); - logger.warn("Snappy native library is available"); - snappyAvailable = true; - boolean hadoopNativeAvailable = NativeCodeLoader.isNativeCodeLoaded(); - loaded = snappyAvailable && hadoopNativeAvailable; - if (loaded) { - logger.info("Snappy native library loaded"); - } else { - logger.warn("Snappy native library not loaded"); - } - } catch (Throwable t) { - logger.warn("Failed to load snappy: ", t); - return false; - } - return loaded; - } - public static CompressDecompressTester of( byte[] rawData) { return new CompressDecompressTester(rawData); @@ -432,7 +411,7 @@ public void assertCompression(String name, Compressor compressor, joiner.join(name, "byte arrays not equals error !!!"), originalRawData, decompressOut.toByteArray()); } catch (Exception ex) { - fail(joiner.join(name, ex.getMessage())); + throw new AssertionError(name + ex, ex); } finally { try { compressedOut.close(); @@ -504,11 +483,10 @@ else if (compressor.getClass().isAssignableFrom(BuiltInZlibDeflater.class) else if (compressor.getClass().isAssignableFrom(ZlibCompressor.class)) { return ZlibFactory.isNativeZlibLoaded(new Configuration()); - } - else if (compressor.getClass().isAssignableFrom(SnappyCompressor.class) - && isNativeSnappyLoadable()) + } else if (compressor.getClass().isAssignableFrom(SnappyCompressor.class)) { return true; - + } + return false; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java index 94ff7a88493c7..462225cebfb67 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java @@ -77,7 +77,6 @@ import org.apache.hadoop.util.ReflectionUtils; import org.junit.After; import org.junit.Assert; -import org.junit.Assume; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,10 +134,8 @@ public void testBZip2NativeCodec() throws IOException { @Test public void testSnappyCodec() throws IOException { - if (SnappyCodec.isNativeCodeLoaded()) { - codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.SnappyCodec"); - codecTest(conf, seed, count, "org.apache.hadoop.io.compress.SnappyCodec"); - } + codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.SnappyCodec"); + codecTest(conf, seed, count, "org.apache.hadoop.io.compress.SnappyCodec"); } @Test @@ -614,7 +611,6 @@ private static void sequenceFileCodecTest(Configuration conf, int lines, */ @Test public void testSnappyMapFile() throws Exception { - Assume.assumeTrue(SnappyCodec.isNativeCodeLoaded()); codecTestMapFile(SnappyCodec.class, CompressionType.BLOCK, 100); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java index 1f035974883cf..43cb4df1105b2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java @@ -27,7 +27,7 @@ import org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; -import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; /** * Test for pairs: @@ -72,20 +72,23 @@ public void testCompressorDecompressor() { } @Test - public void testCompressorDecompressorWithExeedBufferLimit() { - int BYTE_SIZE = 100 * 1024; - byte[] rawData = generate(BYTE_SIZE); + public void testCompressorDecompressorWithExceedBufferLimit() { + // input data size greater than internal buffer size. + final int byteSize = 100 * 1024; + final int bufferSize = 64 * 1024; + byte[] rawData = generate(byteSize); try { CompressDecompressTester.of(rawData) .withCompressDecompressPair( - new SnappyCompressor(BYTE_SIZE + BYTE_SIZE / 2), - new SnappyDecompressor(BYTE_SIZE + BYTE_SIZE / 2)) - .withCompressDecompressPair(new Lz4Compressor(BYTE_SIZE), - new Lz4Decompressor(BYTE_SIZE)) - .withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK, - CompressionTestStrategy.COMPRESS_DECOMPRESS_BLOCK, - CompressionTestStrategy.COMPRESS_DECOMPRESS_ERRORS, - CompressionTestStrategy.COMPRESS_DECOMPRESS_WITH_EMPTY_STREAM)) + new SnappyCompressor(bufferSize), + new SnappyDecompressor(bufferSize)) + .withCompressDecompressPair( + new Lz4Compressor(bufferSize), + new Lz4Decompressor(bufferSize)) + .withTestCases(ImmutableSet.of( + CompressionTestStrategy.COMPRESS_DECOMPRESS_BLOCK, + CompressionTestStrategy.COMPRESS_DECOMPRESS_ERRORS, + CompressionTestStrategy.COMPRESS_DECOMPRESS_WITH_EMPTY_STREAM)) .test(); } catch (Exception ex) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java index c8900bad1df56..93c24835f2206 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java @@ -33,13 +33,13 @@ import java.nio.ByteBuffer; import java.util.Random; +import org.apache.commons.codec.binary.Hex; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.compress.BlockCompressorStream; import org.apache.hadoop.io.compress.BlockDecompressorStream; import org.apache.hadoop.io.compress.CompressionInputStream; import org.apache.hadoop.io.compress.CompressionOutputStream; -import org.apache.hadoop.io.compress.SnappyCodec; import org.apache.hadoop.io.compress.snappy.SnappyDecompressor.SnappyDirectDecompressor; import org.apache.hadoop.test.MultithreadedTestUtil; import org.junit.Assert; @@ -48,8 +48,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.Assume.*; - public class TestSnappyCompressorDecompressor { public static final Logger LOG = @@ -57,7 +55,6 @@ public class TestSnappyCompressorDecompressor { @Before public void before() { - assumeTrue(SnappyCodec.isNativeCodeLoaded()); } @Test @@ -356,8 +353,9 @@ private void compressDecompressLoop(int rawDataSize) throws IOException { @Test public void testSnappyDirectBlockCompression() { - int[] size = { 4 * 1024, 64 * 1024, 128 * 1024, 1024 * 1024 }; - assumeTrue(SnappyCodec.isNativeCodeLoaded()); + int[] size = new int[] { + 4 * 1024, 64 * 1024, 128 * 1024, 1024 * 1024 + }; try { for (int i = 0; i < size.length; i++) { compressDecompressLoop(size[i]); @@ -446,4 +444,52 @@ public void doWork() throws Exception { ctx.waitFor(60000); } + + @Test + public void testSnappyCompatibility() throws Exception { + // HADOOP-17125. Using snappy-java in SnappyCodec. These strings are raw + // data and compressed data using previous native Snappy codec. We use + // updated Snappy codec to decode it and check if it matches. + String rawData = "010a06030a040a0c0109020c0a010204020d02000b010701080605" + + "080b090902060a080502060a0d06070908080a0c0105030904090d050908000" + + "40c090c0d0d0804000d00040b0b0d010d060907020a030a0c09000409050801" + + "07040d0c01060a0b09070a04000b01040b09000e0e00020b06050b060e030e0" + + "a07050d06050d"; + String compressed = "8001f07f010a06030a040a0c0109020c0a010204020d02000b0" + + "10701080605080b090902060a080502060a0d06070908080a0c010503090409" + + "0d05090800040c090c0d0d0804000d00040b0b0d010d060907020a030a0c090" + + "0040905080107040d0c01060a0b09070a04000b01040b09000e0e00020b0605" + + "0b060e030e0a07050d06050d"; + + byte[] rawDataBytes = Hex.decodeHex(rawData); + byte[] compressedBytes = Hex.decodeHex(compressed); + + ByteBuffer inBuf = ByteBuffer.allocateDirect(compressedBytes.length); + inBuf.put(compressedBytes, 0, compressedBytes.length); + inBuf.flip(); + + ByteBuffer outBuf = ByteBuffer.allocateDirect(rawDataBytes.length); + ByteBuffer expected = ByteBuffer.wrap(rawDataBytes); + + SnappyDecompressor.SnappyDirectDecompressor decompressor = + new SnappyDecompressor.SnappyDirectDecompressor(); + + outBuf.clear(); + while(!decompressor.finished()) { + decompressor.decompress(inBuf, outBuf); + if (outBuf.remaining() == 0) { + outBuf.flip(); + while (outBuf.remaining() > 0) { + assertEquals(expected.get(), outBuf.get()); + } + outBuf.clear(); + } + } + outBuf.flip(); + while (outBuf.remaining() > 0) { + assertEquals(expected.get(), outBuf.get()); + } + outBuf.clear(); + assertEquals(0, expected.remaining()); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java index 7e2ab241e195d..ac9ea5e8a8468 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java @@ -42,7 +42,7 @@ import org.apache.hadoop.util.NativeCodeLoader; import org.junit.Before; import org.junit.Test; -import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; public class TestZlibCompressorDecompressor { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java index 6d14de8a521c5..caab72ceff662 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java @@ -66,10 +66,6 @@ public abstract class TestCoderBase { protected boolean allowChangeInputs; - protected int getChunkSize() { - return chunkSize; - } - protected void setChunkSize(int chunkSize) { this.chunkSize = chunkSize; this.zeroChunkBytes = new byte[chunkSize]; // With ZERO by default @@ -90,10 +86,6 @@ protected void prepareBufferAllocator(boolean usingSlicedBuffer) { } } - protected boolean isAllowDump() { - return allowDump; - } - /** * Prepare before running the case. * @param conf diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java index 362bde9806327..631991a03cf9c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.io.erasurecode.rawcoder; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.util.StopWatch; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java index e7130d4da8cb3..6742425e766e8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.ipc; -import com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import org.apache.hadoop.thirdparty.protobuf.BlockingService; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java new file mode 100644 index 0000000000000..fc1057b9f766d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ipc; + +import org.apache.hadoop.conf.Configuration; +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY; + +public class TestCallerContext { + @Test + public void testBuilderAppend() { + Configuration conf = new Configuration(); + conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "$"); + CallerContext.Builder builder = new CallerContext.Builder(null, conf); + CallerContext context = builder.append("context1") + .append("context2").append("key3", "value3").build(); + Assert.assertEquals(true, + context.getContext().contains("$")); + String[] items = context.getContext().split("\\$"); + Assert.assertEquals(3, items.length); + Assert.assertEquals("key3:value3", items[2]); + + builder.append("$$"); + Assert.assertEquals("context1$context2$key3:value3$$$", + builder.build().getContext()); + } + + @Test(expected = IllegalArgumentException.class) + public void testNewBuilder() { + Configuration conf = new Configuration(); + // Set illegal separator. + conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "\t"); + CallerContext.Builder builder = new CallerContext.Builder(null, conf); + builder.build(); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java index 71723325e2c86..a6531b21afce5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java @@ -20,9 +20,15 @@ import static java.lang.Thread.sleep; +import java.util.Map; +import org.eclipse.jetty.util.ajax.JSON; import org.junit.Test; -import static org.junit.Assert.*; +import static org.apache.hadoop.ipc.DecayRpcScheduler.IPC_DECAYSCHEDULER_THRESHOLDS_KEY; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -383,4 +389,93 @@ private int getPriorityIncrementCallCount(String callId) { scheduler.addResponseTime("ignored", mockCall, emptyProcessingDetails); return priority; } + + /** + * Test computing priorities and priority cache of users and service-users. + */ + @Test + public void testServiceUsersCase1() { + Configuration conf = new Configuration(); + conf.setLong("ipc.19." + + DecayRpcScheduler.IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_KEY, 999999); + conf.set("ipc.19." + DecayRpcScheduler.IPC_DECAYSCHEDULER_SERVICE_USERS_KEY, + "service1,service2"); + scheduler = new DecayRpcScheduler(4, "ipc.19", conf); + + assertTrue(scheduler.getServiceUserNames().contains("service1")); + assertTrue(scheduler.getServiceUserNames().contains("service2")); + + for (int i = 0; i < 10; i++) { + getPriorityIncrementCallCount("user1"); + getPriorityIncrementCallCount("service1"); + getPriorityIncrementCallCount("service2"); + } + + assertNotEquals(0, scheduler.getPriorityLevel(mockCall("user1"))); + // The priorities of service users should be always 0. + assertEquals(0, scheduler.getPriorityLevel(mockCall("service1"))); + assertEquals(0, scheduler.getPriorityLevel(mockCall("service2"))); + + // DecayRpcScheduler caches priorities after decay + scheduler.forceDecay(); + // Check priorities on cache + String summary = scheduler.getSchedulingDecisionSummary(); + Map summaryMap = (Map) JSON.parse(summary); + assertNotEquals(0L, summaryMap.get("user1")); + assertEquals(0L, summaryMap.get("service1")); + assertEquals(0L, summaryMap.get("service2")); + } + + /** + * Test the service users' calls are not included when computing user's call + * priority. + */ + @Test + public void testServiceUsersCase2() { + final int level = 4; + Configuration conf = new Configuration(); + conf.setLong("ipc.20." + + DecayRpcScheduler.IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_KEY, 999999); + conf.set("ipc.20." + DecayRpcScheduler.IPC_DECAYSCHEDULER_SERVICE_USERS_KEY, + "service"); + conf.set(IPC_DECAYSCHEDULER_THRESHOLDS_KEY, "0.125,0.25,0.5"); + scheduler = new DecayRpcScheduler(level, "ipc.20", conf); + + // test total costs. + for (int i = 0; i < 10; i++) { + getPriorityIncrementCallCount("user1"); + } + for (int i = 0; i < 50; i++) { + getPriorityIncrementCallCount("service"); + } + assertEquals(10, scheduler.getTotalCallVolume()); + assertEquals(10, scheduler.getTotalRawCallVolume()); + assertEquals(50, scheduler.getTotalServiceUserCallVolume()); + assertEquals(50, scheduler.getTotalServiceUserRawCallVolume()); + // test priority of normal user. + assertEquals(level - 1, scheduler.getPriorityLevel(mockCall("user1"))); + + // test total costs after decay. + scheduler.forceDecay(); + assertEquals(5, scheduler.getTotalCallVolume()); + assertEquals(10, scheduler.getTotalRawCallVolume()); + assertEquals(25, scheduler.getTotalServiceUserCallVolume()); + assertEquals(50, scheduler.getTotalServiceUserRawCallVolume()); + // test priority of normal user. + assertEquals(level - 1, scheduler.getPriorityLevel(mockCall("user1"))); + + // test total costs again. + for (int i = 0; i < 10; i++) { + getPriorityIncrementCallCount("user1"); + } + for (int i = 0; i < 50; i++) { + getPriorityIncrementCallCount("service"); + } + assertEquals(15, scheduler.getTotalCallVolume()); + assertEquals(20, scheduler.getTotalRawCallVolume()); + assertEquals(75, scheduler.getTotalServiceUserCallVolume()); + assertEquals(100, scheduler.getTotalServiceUserRawCallVolume()); + // test priority of normal user. + assertEquals(level - 1, scheduler.getPriorityLevel(mockCall("user1"))); + } } \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java index 2b9828bec6954..32881523fded0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java @@ -99,8 +99,8 @@ import org.mockito.stubbing.Answer; import java.util.function.Supplier; -import com.google.common.primitives.Bytes; -import com.google.common.primitives.Ints; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Bytes; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java index cd2433a8aff10..628c044b1edb3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java @@ -1554,6 +1554,18 @@ public RpcStatusProto getRpcStatusProto() { } } + @Test + public void testSetProtocolEngine() { + Configuration conf = new Configuration(); + RPC.setProtocolEngine(conf, StoppedProtocol.class, StoppedRpcEngine.class); + RpcEngine rpcEngine = RPC.getProtocolEngine(StoppedProtocol.class, conf); + assertTrue(rpcEngine instanceof StoppedRpcEngine); + + RPC.setProtocolEngine(conf, StoppedProtocol.class, ProtobufRpcEngine.class); + rpcEngine = RPC.getProtocolEngine(StoppedProtocol.class, conf); + assertTrue(rpcEngine instanceof StoppedRpcEngine); + } + public static void main(String[] args) throws Exception { new TestRPC().testCallsInternal(conf); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsConfig.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsConfig.java index b53be4d73599a..2ca1c8ad2cc35 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsConfig.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsConfig.java @@ -133,6 +133,22 @@ private void testInstances(MetricsConfig c) throws Exception { assertEq(expected, mc2); } + /** + * Test the config value separated by delimiter + */ + @Test public void testDelimiterConf() { + String filename = getTestFilename("test-metrics2-delimiter"); + new ConfigBuilder().add("p1.foo", "p1foo1,p1foo2,p1foo3").save(filename); + + MetricsConfig mc = MetricsConfig.create("p1", filename); + Configuration expected = new ConfigBuilder() + .add("foo", "p1foo1") + .add("foo", "p1foo2") + .add("foo", "p1foo3") + .config; + assertEq(expected, mc); + } + /** * Return a test filename in the class path * @param basename diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSourceAdapter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSourceAdapter.java index 3fdf445d66447..0dabe468e49e3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSourceAdapter.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSourceAdapter.java @@ -28,7 +28,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsSource; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java index 8ebcc0bb0cca5..5a1f1d1376d4a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java @@ -37,7 +37,7 @@ import static org.mockito.Mockito.*; import java.util.function.Supplier; -import com.google.common.collect.Iterables; +import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables; import org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.metrics2.MetricsException; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/MockDomainNameResolver.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/MockDomainNameResolver.java index 3e436f3a22f8c..3e3bdb7b413b1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/MockDomainNameResolver.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/MockDomainNameResolver.java @@ -23,7 +23,7 @@ import java.util.Map; import java.util.TreeMap; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * This mock resolver class returns the predefined resolving/reverse lookup diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java index 76284932c43de..cfffd85186b89 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java @@ -352,7 +352,7 @@ public void testCreateSocketAddress() throws Throwable { assertEquals(1000, addr.getPort()); try { - addr = NetUtils.createSocketAddr( + NetUtils.createSocketAddr( "127.0.0.1:blahblah", 1000, "myconfig"); fail("Should have failed to parse bad port"); } catch (IllegalArgumentException iae) { @@ -360,6 +360,49 @@ public void testCreateSocketAddress() throws Throwable { } } + @Test + public void testCreateSocketAddressWithURICache() throws Throwable { + InetSocketAddress addr = NetUtils.createSocketAddr( + "127.0.0.1:12345", 1000, "myconfig", true); + assertEquals("127.0.0.1", addr.getAddress().getHostAddress()); + assertEquals(12345, addr.getPort()); + + addr = NetUtils.createSocketAddr( + "127.0.0.1:12345", 1000, "myconfig", true); + assertEquals("127.0.0.1", addr.getAddress().getHostAddress()); + assertEquals(12345, addr.getPort()); + + // ---------------------------------------------------- + + addr = NetUtils.createSocketAddr( + "127.0.0.1", 1000, "myconfig", true); + assertEquals("127.0.0.1", addr.getAddress().getHostAddress()); + assertEquals(1000, addr.getPort()); + + addr = NetUtils.createSocketAddr( + "127.0.0.1", 1000, "myconfig", true); + assertEquals("127.0.0.1", addr.getAddress().getHostAddress()); + assertEquals(1000, addr.getPort()); + + // ---------------------------------------------------- + + try { + NetUtils.createSocketAddr( + "127.0.0.1:blahblah", 1000, "myconfig", true); + fail("Should have failed to parse bad port"); + } catch (IllegalArgumentException iae) { + assertInException(iae, "myconfig"); + } + + try { + NetUtils.createSocketAddr( + "127.0.0.1:blahblah", 1000, "myconfig", true); + fail("Should have failed to parse bad port"); + } catch (IllegalArgumentException iae) { + assertInException(iae, "myconfig"); + } + } + private void assertRemoteDetailsIncluded(IOException wrapped) throws Throwable { assertInException(wrapped, "desthost"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java index 4b8b02f0171d4..86870e1257119 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java @@ -21,8 +21,8 @@ import static org.junit.Assert.assertEquals; -import com.google.common.base.Charsets; -import com.google.common.io.Files; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.io.Files; import java.io.File; import java.io.IOException; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java index c0d204f86aa82..466c83eb58212 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java @@ -49,7 +49,7 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; -import com.google.common.io.Files; +import org.apache.hadoop.thirdparty.com.google.common.io.Files; public class TestDomainSocket { private static TemporarySocketDirectory sockDir; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java index aa522f266014b..ca801dac2c247 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java @@ -31,7 +31,7 @@ import org.junit.Before; import org.junit.Test; -import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java index dba2b086f897f..aba3997187747 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java @@ -389,8 +389,9 @@ public void run() { fail("The LDAP query should have timed out!"); } catch (NamingException ne) { LOG.debug("Got the exception while LDAP querying: ", ne); - assertExceptionContains("LDAP response read timed out, timeout used:" + - connectionTimeoutMs + "ms", ne); + assertExceptionContains("LDAP response read timed out, timeout used", + ne); + assertExceptionContains("" + connectionTimeoutMs, ne); assertFalse(ne.getMessage().contains("remaining name")); } finally { finLatch.countDown(); @@ -444,8 +445,9 @@ public void run() { fail("The LDAP query should have timed out!"); } catch (NamingException ne) { LOG.debug("Got the exception while LDAP querying: ", ne); - assertExceptionContains("LDAP response read timed out, timeout used:" + - readTimeoutMs + "ms", ne); + assertExceptionContains("LDAP response read timed out, timeout used", + ne); + assertExceptionContains(""+ readTimeoutMs, ne); assertExceptionContains("remaining name", ne); } finally { finLatch.countDown(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithBindUserSwitch.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithBindUserSwitch.java index 4f080b3fede96..3b4c77d9e4ab9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithBindUserSwitch.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithBindUserSwitch.java @@ -18,7 +18,7 @@ package org.apache.hadoop.security; -import com.google.common.collect.Iterators; +import org.apache.hadoop.thirdparty.com.google.common.collect.Iterators; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.alias.CredentialProvider; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java index 016c589ae3a24..b6b9684445342 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java @@ -46,7 +46,7 @@ import org.junit.Test; import org.mockito.Mockito; -import com.google.common.io.Files; +import org.apache.hadoop.thirdparty.com.google.common.io.Files; public class TestSecurityUtil { @@ -370,6 +370,16 @@ public void testSocketAddrWithNameToStaticIP() { verifyServiceAddr(staticHost, "255.255.255.255"); } + @Test + public void testSocketAddrWithChangeIP() { + String staticHost = "host4"; + NetUtils.addStaticResolution(staticHost, "255.255.255.255"); + verifyServiceAddr(staticHost, "255.255.255.255"); + + NetUtils.addStaticResolution(staticHost, "255.255.255.254"); + verifyServiceAddr(staticHost, "255.255.255.254"); + } + // this is a bizarre case, but it's if a test tries to remap an ip address @Test public void testSocketAddrWithIPToStaticIP() { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java index e6fdc2bcdfbd4..939209d267b50 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java @@ -32,8 +32,8 @@ import org.apache.hadoop.security.ShellBasedIdMapping.StaticMapping; import org.junit.Test; -import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.BiMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.HashBiMap; import static org.assertj.core.api.Assertions.assertThat; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java index 8ede451db964c..db0095f2171e2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java @@ -23,6 +23,7 @@ import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.Time; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -101,12 +102,35 @@ public void stopMiniKdc() { } } + /** + * Login from keytab using the MiniKDC. + */ + @Test + public void testUGILoginFromKeytab() throws Exception { + long beforeLogin = Time.now(); + String principal = "foo"; + File keytab = new File(workDir, "foo.keytab"); + kdc.createPrincipal(keytab, principal); + + UserGroupInformation.loginUserFromKeytab(principal, keytab.getPath()); + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + Assert.assertTrue("UGI should be configured to login from keytab", + ugi.isFromKeytab()); + + User user = getUser(ugi.getSubject()); + Assert.assertNotNull(user.getLogin()); + + Assert.assertTrue("User login time is less than before login time, " + + "beforeLoginTime:" + beforeLogin + " userLoginTime:" + user.getLastLogin(), + user.getLastLogin() > beforeLogin); + } + /** * Login from keytab using the MiniKDC and verify the UGI can successfully * relogin from keytab as well. This will catch regressions like HADOOP-10786. */ @Test - public void testUGILoginFromKeytab() throws Exception { + public void testUGIReLoginFromKeytab() throws Exception { String principal = "foo"; File keytab = new File(workDir, "foo.keytab"); kdc.createPrincipal(keytab, principal); @@ -122,6 +146,9 @@ public void testUGILoginFromKeytab() throws Exception { final LoginContext login1 = user.getLogin(); Assert.assertNotNull(login1); + // Sleep for 2 secs to have a difference between first and second login + Thread.sleep(2000); + ugi.reloginFromKeytab(); final long secondLogin = user.getLastLogin(); final LoginContext login2 = user.getLogin(); @@ -131,6 +158,42 @@ public void testUGILoginFromKeytab() throws Exception { Assert.assertNotSame(login1, login2); } + /** + * Force re-login from keytab using the MiniKDC and verify the UGI can + * successfully relogin from keytab as well. + */ + @Test + public void testUGIForceReLoginFromKeytab() throws Exception { + // Set this to false as we are testing force re-login anyways + UserGroupInformation.setShouldRenewImmediatelyForTests(false); + String principal = "foo"; + File keytab = new File(workDir, "foo.keytab"); + kdc.createPrincipal(keytab, principal); + + UserGroupInformation.loginUserFromKeytab(principal, keytab.getPath()); + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + Assert.assertTrue("UGI should be configured to login from keytab", + ugi.isFromKeytab()); + + // Verify relogin from keytab. + User user = getUser(ugi.getSubject()); + final long firstLogin = user.getLastLogin(); + final LoginContext login1 = user.getLogin(); + Assert.assertNotNull(login1); + + // Sleep for 2 secs to have a difference between first and second login + Thread.sleep(2000); + + // Force relogin from keytab + ugi.forceReloginFromKeytab(); + final long secondLogin = user.getLastLogin(); + final LoginContext login2 = user.getLogin(); + Assert.assertTrue("User should have been able to relogin from keytab", + secondLogin > firstLogin); + Assert.assertNotNull(login2); + Assert.assertNotSame(login1, login2); + } + @Test public void testGetUGIFromKnownSubject() throws Exception { KerberosPrincipal principal = new KerberosPrincipal("user"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java index ee7e42cb1daf0..f7bb8ec4a964b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java @@ -36,6 +36,7 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +52,9 @@ public class TestCredentialProviderFactory { @Rule public final TestName test = new TestName(); + @Rule + public ExpectedException exception = ExpectedException.none(); + @Before public void announce() { LOG.info("Running test " + test.getMethodName()); @@ -245,6 +249,21 @@ public void testLocalJksProvider() throws Exception { checkPermissionRetention(conf, ourUrl, path); } + @Test + public void testLocalBCFKSProvider() throws Exception { + Configuration conf = new Configuration(); + final Path ksPath = new Path(tmpDir.toString(), "test.bcfks"); + final String ourUrl = LocalBouncyCastleFipsKeyStoreProvider.SCHEME_NAME + + "://file" + ksPath.toUri(); + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl); + + exception.expect(IOException.class); + exception.expectMessage("Can't create keystore"); + List providers = + CredentialProviderFactory.getProviders(conf); + assertTrue("BCFKS needs additional JDK setup", providers.isEmpty()); + } + public void checkPermissionRetention(Configuration conf, String ourUrl, Path path) throws Exception { CredentialProvider provider = CredentialProviderFactory.getProviders(conf).get(0); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java index 9b4d1f205ff58..4e5a6fbd7e0f9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java @@ -17,7 +17,14 @@ */ package org.apache.hadoop.security.ssl; +import static org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY; import static org.apache.hadoop.security.ssl.KeyStoreTestUtil.TRUST_STORE_PASSWORD_DEFAULT; +import static org.apache.hadoop.security.ssl.SSLFactory.Mode.CLIENT; +import static org.apache.hadoop.security.ssl.SSLFactory.SSL_CLIENT_CONF_KEY; +import static org.apache.hadoop.security.ssl.SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import org.apache.hadoop.conf.Configuration; @@ -66,6 +73,10 @@ public class TestSSLFactory { + "SSL_RSA_EXPORT_WITH_DES40_CBC_SHA," + "SSL_RSA_WITH_RC4_128_MD5," + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA"; + private static final Configuration FAKE_SSL_CONFIG = + KeyStoreTestUtil.createClientSSLConfig("clientKeyStoreLocation", + "keystorePassword", "keyPassword", + "trustStoreLocation", "trustStorePassword"); @BeforeClass public static void setUp() throws Exception { @@ -90,6 +101,55 @@ public void cleanUp() throws Exception { KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir); } + private String getClientTrustStoreKeyName() { + return FileBasedKeyStoresFactory.resolvePropertyName( + CLIENT, SSL_TRUSTSTORE_LOCATION_TPL_KEY); + } + + @Test + public void testNonExistSslClientXml() throws Exception{ + Configuration conf = new Configuration(false); + conf.setBoolean(SSL_REQUIRE_CLIENT_CERT_KEY, false); + conf.set(SSL_CLIENT_CONF_KEY, "non-exist-ssl-client.xml"); + Configuration sslConf = + SSLFactory.readSSLConfiguration(conf, SSLFactory.Mode.CLIENT); + assertNull(sslConf.getResource("non-exist-ssl-client.xml")); + assertNull(sslConf.get("ssl.client.truststore.location")); + } + + @Test + public void testSslConfFallback() throws Exception { + Configuration conf = new Configuration(FAKE_SSL_CONFIG); + + // Set non-exist-ssl-client.xml that fails to load. + // This triggers fallback to SSL config from input conf. + conf.set(SSL_CLIENT_CONF_KEY, "non-exist-ssl-client.xml"); + Configuration sslConf = SSLFactory.readSSLConfiguration(conf, CLIENT); + + // Verify fallback to input conf when ssl conf can't be loaded from + // classpath. + String clientTsLoc = sslConf.get(getClientTrustStoreKeyName()); + assertEquals("trustStoreLocation", clientTsLoc); + assertEquals(sslConf, conf); + } + + @Test + public void testSslConfClassPathFirst() throws Exception { + // Generate a valid ssl-client.xml into classpath. + // This will be the preferred approach. + Configuration conf = createConfiguration(false, true); + + // Injecting fake ssl config into input conf. + conf.addResource(FAKE_SSL_CONFIG); + + // Classpath SSL config will be preferred if both input conf and + // the classpath SSL config exist for backward compatibility. + Configuration sslConfLoaded = SSLFactory.readSSLConfiguration(conf, CLIENT); + String clientTsLoc = sslConfLoaded.get(getClientTrustStoreKeyName()); + assertNotEquals("trustStoreLocation", clientTsLoc); + assertNotEquals(conf, sslConfLoaded); + } + @Test(expected = IllegalStateException.class) public void clientMode() throws Exception { Configuration conf = createConfiguration(false, true); @@ -452,7 +512,7 @@ private void checkSSLFactoryInitWithPasswords(SSLFactory.Mode mode, // Create the master configuration for use by the SSLFactory, which by // default refers to the ssl-server.xml or ssl-client.xml created above. Configuration conf = new Configuration(); - conf.setBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, true); + conf.setBoolean(SSL_REQUIRE_CLIENT_CERT_KEY, true); // Try initializing an SSLFactory. SSLFactory sslFactory = new SSLFactory(mode, conf); @@ -466,7 +526,7 @@ private void checkSSLFactoryInitWithPasswords(SSLFactory.Mode mode, @Test public void testNoClientCertsInitialization() throws Exception { Configuration conf = createConfiguration(false, true); - conf.unset(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY); + conf.unset(SSL_REQUIRE_CLIENT_CERT_KEY); SSLFactory sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); try { sslFactory.init(); @@ -478,7 +538,7 @@ public void testNoClientCertsInitialization() throws Exception { @Test public void testNoTrustStore() throws Exception { Configuration conf = createConfiguration(false, false); - conf.unset(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY); + conf.unset(SSL_REQUIRE_CLIENT_CERT_KEY); SSLFactory sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf); try { sslFactory.init(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/AssertExtensions.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/AssertExtensions.java new file mode 100644 index 0000000000000..8c5e553f71ee3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/AssertExtensions.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.test; + +import java.util.concurrent.Callable; + +import org.assertj.core.description.Description; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Extra classes to work with AssertJ. + * These are kept separate from {@link LambdaTestUtils} so there's + * no requirement for AssertJ to be on the classpath in that broadly + * used class. + */ +public final class AssertExtensions { + + private static final Logger LOG = + LoggerFactory.getLogger(AssertExtensions.class); + + private AssertExtensions() { + } + + /** + * A description for AssertJ "describedAs" clauses which evaluates the + * lambda-expression only on failure. That must return a string + * or null/"" to be skipped. + * @param eval lambda expression to invoke + * @return a description for AssertJ + */ + public static Description dynamicDescription(Callable eval) { + return new DynamicDescription(eval); + } + + private static final class DynamicDescription extends Description { + private final Callable eval; + + private DynamicDescription(final Callable eval) { + this.eval = eval; + } + + @Override + public String value() { + try { + return eval.call(); + } catch (Exception e) { + LOG.warn("Failed to evaluate description: " + e); + LOG.debug("Evaluation failure", e); + // return null so that the description evaluation chain + // will skip this one + return null; + } + } + } + + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java index 79fc0083a0cc1..ade6cb4c7b8a0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java @@ -62,8 +62,8 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import com.google.common.base.Joiner; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; /** * Test provides some very generic helpers which might be used across the tests diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java index ad265afc3a022..3e0d31dc6a150 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java @@ -18,7 +18,7 @@ package org.apache.hadoop.test; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java index 0cb5288e881ff..eb8d938994735 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java @@ -18,7 +18,7 @@ package org.apache.hadoop.test; -import static com.google.common.base.Preconditions.*; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*; import org.junit.Assert; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/JarFinder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/JarFinder.java index 478a29b3317b4..5e0bfc2399270 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/JarFinder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/JarFinder.java @@ -13,7 +13,7 @@ */ package org.apache.hadoop.util; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java index 570e54214ad54..0fb887676274a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java @@ -41,8 +41,8 @@ import org.junit.Before; import org.junit.Test; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Splitter; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; public class TestApplicationClassLoader { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDirectBufferPool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDirectBufferPool.java index c8fd754666c9e..d6da2f86cc3c4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDirectBufferPool.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDirectBufferPool.java @@ -26,7 +26,7 @@ import org.junit.Test; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; public class TestDirectBufferPool { final org.apache.hadoop.util.DirectBufferPool pool = new org.apache.hadoop.util.DirectBufferPool(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java index 0dbfe3d6cdd36..592b0bdc4483e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java @@ -50,7 +50,7 @@ import org.junit.Before; import org.junit.Test; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; public class TestGenericOptionsParser { File testDir; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java index 58874fdcdfba6..d3da6c191071d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java @@ -22,7 +22,6 @@ import org.apache.hadoop.crypto.OpensslCipher; import org.apache.hadoop.io.compress.Lz4Codec; -import org.apache.hadoop.io.compress.SnappyCodec; import org.apache.hadoop.io.compress.zlib.ZlibFactory; import org.apache.hadoop.util.NativeCodeLoader; import org.slf4j.Logger; @@ -52,9 +51,6 @@ public void testNativeCodeLoaded() { // library names are depended on platform and build envs // so just check names are available assertFalse(ZlibFactory.getLibraryName().isEmpty()); - if (NativeCodeLoader.buildSupportsSnappy()) { - assertFalse(SnappyCodec.getLibraryName().isEmpty()); - } if (NativeCodeLoader.buildSupportsOpenssl()) { assertFalse(OpensslCipher.getLibraryName().isEmpty()); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestReflectionUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestReflectionUtils.java index 62cd62506f115..03c67c8263938 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestReflectionUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestReflectionUtils.java @@ -28,6 +28,7 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.*; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils.LogCapturer; import org.junit.Before; import org.junit.Test; @@ -168,6 +169,34 @@ public void testLogThreadInfo() throws Exception { containsString("Process Thread Dump: " + title)); } + @Test + public void testNewInstanceForNonDefaultConstructor() { + Object x = ReflectionUtils.newInstance( + NoDefaultCtor.class, null, new Class[] {int.class}, 1); + assertTrue(x instanceof NoDefaultCtor); + } + + @Test + public void testNewInstanceForNonDefaultConstructorWithException() { + try { + ReflectionUtils.newInstance( + NoDefaultCtor.class, null, new Class[]{int.class}, 1, 2); + fail("Should have failed before this point"); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains( + "1 parameters are required but 2 arguments are provided", e); + } + + try { + ReflectionUtils.newInstance( + NoDefaultCtor.class, null, new Class[]{int.class}); + fail("Should have failed before this point"); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains( + "1 parameters are required but 0 arguments are provided", e); + } + } + // Used for testGetDeclaredFieldsIncludingInherited private class Parent { private int parentField; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java index 3d985e40fb878..93790eb1350a7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java @@ -31,8 +31,8 @@ import org.apache.zookeeper.data.ACL; import org.junit.Test; -import com.google.common.base.Charsets; -import com.google.common.io.Files; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.io.Files; public class TestZKUtil { private static final String TEST_ROOT_DIR = GenericTestUtils.getTempPath( diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/BloomFilterCommonTester.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/BloomFilterCommonTester.java index ca018a64c5e81..f43930dd07a5e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/BloomFilterCommonTester.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/BloomFilterCommonTester.java @@ -33,8 +33,8 @@ import org.apache.hadoop.util.hash.Hash; import org.apache.log4j.Logger; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; public class BloomFilterCommonTester { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java index fbbb6d81935cd..cfd9628885d4f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java @@ -31,9 +31,9 @@ import org.junit.Assert; import org.junit.Test; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; public class TestBloomFilters { diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index acfdeeac50b0a..338af127f8bd7 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -54,8 +54,8 @@ compile - com.google.guava - guava + org.apache.hadoop.thirdparty + hadoop-shaded-guava compile diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java index b6b425443babc..4788dfd8755f7 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.crypto.key.kms.server; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.util.KMSUtil; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.classification.InterfaceAudience; @@ -52,10 +52,12 @@ import java.net.URI; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.EnumSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import static org.apache.hadoop.crypto.key.kms.server.KMSACLs.INVALIDATE_CACHE_TYPES; import static org.apache.hadoop.util.KMSUtil.checkNotEmpty; import static org.apache.hadoop.util.KMSUtil.checkNotNull; @@ -95,6 +97,12 @@ private void assertAccess(KMSACLs.Type aclType, UserGroupInformation ugi, KMSWebApp.getACLs().assertAccess(aclType, ugi, operation, key); } + private void assertAccess(EnumSet aclTypes, + UserGroupInformation ugi, KMSOp operation, String key) + throws AccessControlException { + KMSWebApp.getACLs().assertAccess(aclTypes, ugi, operation, key); + } + private static KeyProvider.KeyVersion removeKeyMaterial( KeyProvider.KeyVersion keyVersion) { return new KMSClientProvider.KMSKeyVersion(keyVersion.getName(), @@ -270,7 +278,7 @@ public Response invalidateCache(@PathParam("name") final String name) KMSWebApp.getAdminCallsMeter().mark(); checkNotEmpty(name, "name"); UserGroupInformation user = HttpUserGroupInformation.get(); - assertAccess(KMSACLs.Type.ROLLOVER, user, KMSOp.INVALIDATE_CACHE, name); + assertAccess(INVALIDATE_CACHE_TYPES, user, KMSOp.INVALIDATE_CACHE, name); LOG.debug("Invalidating cache with key name {}.", name); user.doAs(new PrivilegedExceptionAction() { diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java index ba0fe825b4eb1..f310b73b614b7 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java @@ -29,13 +29,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.EnumSet; import java.util.HashMap; import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Provides access to the AccessControlLists used by KMS, @@ -69,6 +70,10 @@ public String getBlacklistConfigKey() { public static final int RELOADER_SLEEP_MILLIS = 1000; + // Allow both ROLLOVER and DELETE to invalidate cache. + public static final EnumSet INVALIDATE_CACHE_TYPES = + EnumSet.of(KMSACLs.Type.ROLLOVER, KMSACLs.Type.DELETE); + private volatile Map acls; private volatile Map blacklistedAcls; @VisibleForTesting @@ -273,6 +278,27 @@ public void assertAccess(KMSACLs.Type aclType, } } + public void assertAccess(EnumSet aclTypes, + UserGroupInformation ugi, KMSOp operation, String key) + throws AccessControlException { + boolean accessAllowed = false; + for (KMSACLs.Type type : aclTypes) { + if (KMSWebApp.getACLs().hasAccess(type, ugi)){ + accessAllowed = true; + break; + } + } + + if (!accessAllowed) { + KMSWebApp.getUnauthorizedCallsMeter().mark(); + KMSWebApp.getKMSAudit().unauthorized(ugi, operation, key); + throw new AuthorizationException(String.format( + (key != null) ? UNAUTHORIZED_MSG_WITH_KEY + : UNAUTHORIZED_MSG_WITHOUT_KEY, + ugi.getShortUserName(), operation, key)); + } + } + @Override public boolean hasAccessToKey(String keyName, UserGroupInformation ugi, KeyOpType opType) { diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java index 13a2d5c57a74e..7ea1f4feab018 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java @@ -20,8 +20,8 @@ import static org.apache.hadoop.crypto.key.kms.server.KMSAuditLogger.AuditEvent; import static org.apache.hadoop.crypto.key.kms.server.KMSAuditLogger.OpStatus; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.kms.server.KMSACLs.Type; import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyOpType; @@ -31,13 +31,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Strings; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; -import com.google.common.collect.Sets; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.base.Strings; +import org.apache.hadoop.thirdparty.com.google.common.cache.Cache; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalListener; +import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalNotification; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.HashSet; import java.util.concurrent.Callable; diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java index da542ffb191e6..c020af27b96aa 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.crypto.key.kms.server; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.kms.KMSDelegationToken; diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java index f3c0bbdda6183..dc2ba3261cfd8 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java @@ -21,7 +21,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import javax.servlet.Filter; import javax.servlet.FilterChain; diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java index da597b4da5f81..a92dd1045c01a 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java @@ -26,7 +26,7 @@ import com.codahale.metrics.JmxReporter; import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.CachingKeyProvider; diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java index 101591b0310d2..fe3207b31c27a 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java @@ -32,9 +32,9 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Strings; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; /** * A {@link KeyProvider} proxy that checks whether the current user derived via diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/SimpleKMSAuditLogger.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/SimpleKMSAuditLogger.java index 4dcbe2c54f2f7..74825097f3038 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/SimpleKMSAuditLogger.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/SimpleKMSAuditLogger.java @@ -23,8 +23,8 @@ import java.util.LinkedList; import java.util.List; -import com.google.common.base.Joiner; -import com.google.common.base.Strings; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm index 5490219750020..95e926b3561a1 100644 --- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm +++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm @@ -307,7 +307,7 @@ Configure `etc/hadoop/ssl-server.xml` with proper values, for example: ``` The SSL passwords can be secured by a credential provider. See -[Credential Provider API](../../../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html). +[Credential Provider API](../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html). You need to create an SSL certificate for the KMS. As the `kms` Unix user, using the Java `keytool` command to create the SSL certificate: @@ -716,7 +716,7 @@ $H4 HTTP Kerberos Principals Configuration When KMS instances are behind a load-balancer or VIP, clients will use the hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the URL is used to construct the Kerberos service name of the server, `HTTP/#HOSTNAME#`. This means that all KMS instances must have a Kerberos service name with the load-balancer or VIP hostname. -In order to be able to access directly a specific KMS instance, the KMS instance must also have Keberos service name with its own hostname. This is required for monitoring and admin purposes. +In order to be able to access directly a specific KMS instance, the KMS instance must also have Kerberos service name with its own hostname. This is required for monitoring and admin purposes. Both Kerberos service principal credentials (for the load-balancer/VIP hostname and for the actual KMS instance hostname) must be in the keytab file configured for authentication. And the principal name specified in the configuration must be '\*'. For example: diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java index faa8fa280a9cf..bc4bbc3df70bd 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java @@ -26,7 +26,7 @@ import java.io.Writer; import java.net.URL; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index 5c18c772a1fe8..3d59e6f5be7b7 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -18,7 +18,7 @@ package org.apache.hadoop.crypto.key.kms.server; import java.util.function.Supplier; -import com.google.common.cache.LoadingCache; +import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; import org.apache.curator.test.TestingServer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProviderFactory; diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml index 22c56722d1c42..b3c727910e7c2 100644 --- a/hadoop-common-project/hadoop-nfs/pom.xml +++ b/hadoop-common-project/hadoop-nfs/pom.xml @@ -94,8 +94,8 @@ compile - com.google.guava - guava + org.apache.hadoop.thirdparty + hadoop-shaded-guava org.assertj diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/NfsExports.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/NfsExports.java index 3d5088d7006c4..97b8a444ac28a 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/NfsExports.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/NfsExports.java @@ -32,7 +32,7 @@ import org.apache.hadoop.util.LightWeightGSet.LinkedElement; import org.apache.hadoop.util.StringUtils; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LOOKUP3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LOOKUP3Request.java index 4d31a8203f24e..46e4f7259d808 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LOOKUP3Request.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LOOKUP3Request.java @@ -23,7 +23,7 @@ import org.apache.hadoop.nfs.nfs3.FileHandle; import org.apache.hadoop.oncrpc.XDR; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * LOOKUP3 Request diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/READ3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/READ3Request.java index 5898ec588ff31..e2f3e2ff8e03a 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/READ3Request.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/READ3Request.java @@ -22,7 +22,7 @@ import org.apache.hadoop.nfs.nfs3.FileHandle; import org.apache.hadoop.oncrpc.XDR; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * READ3 Request diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIR3Response.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIR3Response.java index 5bde2c0f69a8f..6fbfd5f0c6671 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIR3Response.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIR3Response.java @@ -28,7 +28,7 @@ import org.apache.hadoop.oncrpc.XDR; import org.apache.hadoop.oncrpc.security.Verifier; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * READDIR3 Response diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIRPLUS3Response.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIRPLUS3Response.java index cf32bd1c87865..5e814c488e7cd 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIRPLUS3Response.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIRPLUS3Response.java @@ -30,7 +30,7 @@ import org.apache.hadoop.oncrpc.XDR; import org.apache.hadoop.oncrpc.security.Verifier; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * READDIRPLUS3 Response diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcCallCache.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcCallCache.java index 2d6f30ecb56cb..8632a387c6032 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcCallCache.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcCallCache.java @@ -23,7 +23,7 @@ import java.util.Map; import java.util.Map.Entry; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * This class is used for handling the duplicate non-idempotenty Rpc diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java index 5c059aa4550a4..d4b2261e05f22 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java @@ -22,7 +22,7 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.oncrpc.RpcAcceptedReply.AcceptState; import org.apache.hadoop.oncrpc.security.Verifier; import org.apache.hadoop.oncrpc.security.VerifierNone; diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcReply.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcReply.java index e866a5c419cd3..985629e0285cb 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcReply.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcReply.java @@ -20,7 +20,7 @@ import org.apache.hadoop.oncrpc.security.RpcAuthInfo; import org.apache.hadoop.oncrpc.security.Verifier; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Represents an RPC message of type RPC reply as defined in RFC 1831 diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java index acdc51c954521..419eff831f0e7 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java @@ -23,8 +23,8 @@ import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Utility class for building XDR messages based on RFC 4506. diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/Credentials.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/Credentials.java index 64edf485b29f7..fd832c4ab2409 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/Credentials.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/Credentials.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.oncrpc.security; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.oncrpc.XDR; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsNone.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsNone.java index 753edba49fbda..f62dc6bd223b7 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsNone.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsNone.java @@ -19,7 +19,7 @@ import org.apache.hadoop.oncrpc.XDR; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** Credential used by AUTH_NONE */ public class CredentialsNone extends Credentials { diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java index 19ba32022eef7..8713d210f4678 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java @@ -21,7 +21,7 @@ import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.oncrpc.XDR; /** Credential used by AUTH_SYS */ diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/VerifierNone.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/VerifierNone.java index 8bccd1b9be247..1f8ad7b37138b 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/VerifierNone.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/VerifierNone.java @@ -19,7 +19,7 @@ import org.apache.hadoop.oncrpc.XDR; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** Verifier used by AUTH_NONE. */ public class VerifierNone extends Verifier { diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java index 123999d5e14c0..80f43828ea83b 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java @@ -38,7 +38,7 @@ import org.jboss.netty.handler.timeout.IdleStateHandler; import org.jboss.netty.util.HashedWheelTimer; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-registry/pom.xml b/hadoop-common-project/hadoop-registry/pom.xml index d5e0150bba94c..5e3e2edcb9690 100644 --- a/hadoop-common-project/hadoop-registry/pom.xml +++ b/hadoop-common-project/hadoop-registry/pom.xml @@ -126,8 +126,8 @@ - com.google.guava - guava + org.apache.hadoop.thirdparty + hadoop-shaded-guava diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/cli/RegistryCli.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/cli/RegistryCli.java index 480ce0ed5fb20..a1349f3e26f00 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/cli/RegistryCli.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/cli/RegistryCli.java @@ -27,7 +27,7 @@ import java.util.List; import java.util.Map; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperationsFactory.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperationsFactory.java index 1a8bb3ec0262a..8a26b4b450def 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperationsFactory.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperationsFactory.java @@ -18,7 +18,7 @@ package org.apache.hadoop.registry.client.api; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.registry.server.dns.RegistryDNS; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java index 5f9c5f37508c1..786bec040b22d 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java @@ -18,7 +18,7 @@ package org.apache.hadoop.registry.client.api; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.ServiceStateException; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java index b8e9ba1bd7fbe..09df00d083c3e 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java @@ -18,7 +18,7 @@ package org.apache.hadoop.registry.client.binding; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.PathNotFoundException; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java index 05df3255e3a76..9a4369cdda385 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java @@ -18,7 +18,7 @@ package org.apache.hadoop.registry.client.binding; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.registry.client.exceptions.InvalidRecordException; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java index 1b839c253b14d..d862fe649b5ac 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java @@ -18,8 +18,8 @@ package org.apache.hadoop.registry.client.binding; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/FSRegistryOperationsService.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/FSRegistryOperationsService.java index 41884a984844f..6a08dcc074725 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/FSRegistryOperationsService.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/FSRegistryOperationsService.java @@ -47,8 +47,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Filesystem-based implementation of RegistryOperations. This class relies diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java index 2eb7aa54f09bc..a01b7151b6989 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java @@ -18,8 +18,8 @@ package org.apache.hadoop.registry.client.impl.zk; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.curator.ensemble.EnsembleProvider; import org.apache.curator.ensemble.fixed.FixedEnsembleProvider; import org.apache.curator.framework.CuratorFramework; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java index 4c911da156b8e..e46a016baa07d 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java @@ -18,7 +18,7 @@ package org.apache.hadoop.registry.client.impl.zk; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.registry.client.api.BindFlags; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java index c3cb021fb53f3..065cbe3296b09 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java @@ -18,9 +18,9 @@ package org.apache.hadoop.registry.client.impl.zk; -import com.google.common.base.Preconditions; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Splitter; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/ZKPathDumper.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/ZKPathDumper.java index 3c4a730608f6e..e045c16e84b0a 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/ZKPathDumper.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/ZKPathDumper.java @@ -18,8 +18,8 @@ package org.apache.hadoop.registry.client.impl.zk; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.GetChildrenBuilder; import org.apache.zookeeper.data.ACL; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/types/Endpoint.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/types/Endpoint.java index 392884faf87f5..b92b93df7be55 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/types/Endpoint.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/types/Endpoint.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.registry.client.binding.JsonSerDeser; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecord.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecord.java index 9bb02c3cc3816..1a85436ed17ef 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecord.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecord.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonInclude; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java index be63d028f1fd5..eeee581540963 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java @@ -16,7 +16,7 @@ */ package org.apache.hadoop.registry.server.dns; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.IOFileFilter; import org.apache.commons.net.util.Base64; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java index 826b02c5e513c..1ff5f26b47207 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java @@ -16,7 +16,7 @@ */ package org.apache.hadoop.registry.server.dns; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.registry.client.api.DNSOperationsFactory; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/dns/ReverseZoneUtils.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/dns/ReverseZoneUtils.java index 796f46bae108e..bb375831d6eae 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/dns/ReverseZoneUtils.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/dns/ReverseZoneUtils.java @@ -16,7 +16,7 @@ */ package org.apache.hadoop.registry.server.dns; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.net.Inet6Address; import java.net.InetAddress; import java.net.UnknownHostException; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/integration/SelectByYarnPersistence.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/integration/SelectByYarnPersistence.java index 6a1993eafd9c1..8d395e4c5c763 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/integration/SelectByYarnPersistence.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/integration/SelectByYarnPersistence.java @@ -18,7 +18,7 @@ package org.apache.hadoop.registry.server.integration; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.registry.client.types.RegistryPathStatus; import org.apache.hadoop.registry.client.types.ServiceRecord; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/services/MicroZookeeperService.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/services/MicroZookeeperService.java index a7e2611b3df9d..0ab4cd2f3bfac 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/services/MicroZookeeperService.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/services/MicroZookeeperService.java @@ -18,7 +18,7 @@ package org.apache.hadoop.registry.server.services; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; import org.apache.curator.ensemble.fixed.FixedEnsembleProvider; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/services/RegistryAdminService.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/services/RegistryAdminService.java index d60797e71ea02..3234088e01d84 100644 --- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/services/RegistryAdminService.java +++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/services/RegistryAdminService.java @@ -19,7 +19,7 @@ package org.apache.hadoop.registry.server.services; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.api.BackgroundCallback; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml index 278d01dc22d0f..c96b3a99bd1c4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml @@ -22,6 +22,7 @@ + diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java index 4162b198fb124..b014222fea5fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -153,7 +153,19 @@ public FileStatus getFileStatus(Path f) throw new FileNotFoundException("File does not exist: " + f.toString()); } } - + + /** + * Synchronize client metadata state with Active NameNode. + *

+ * In HA the client synchronizes its state with the Active NameNode + * in order to guarantee subsequent read consistency from Observer Nodes. + * @throws IOException + */ + @Override + public void msync() throws IOException { + dfs.msync(); + } + @Override public FileStatus getFileLinkStatus(Path f) throws IOException, UnresolvedLinkException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java index 7a03240e80d9c..b34420da5cec3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java @@ -43,7 +43,7 @@ import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.ReflectionUtils; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java index 9b324bd1b07ab..4de969642d574 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java @@ -29,7 +29,7 @@ /** * Global State Id context for the client. - *

+ *

* This is the client side implementation responsible for receiving * state alignment info from server(s). */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 5a6a0f65f12f6..d9f1b46a4506d 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -150,6 +150,7 @@ import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil; import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair; import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure; @@ -192,11 +193,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.net.InetAddresses; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses; /******************************************************** * DFSClient can connect to a Hadoop Filesystem and @@ -244,7 +245,6 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, new DFSHedgedReadMetrics(); private static ThreadPoolExecutor HEDGED_READ_THREAD_POOL; private static volatile ThreadPoolExecutor STRIPED_READ_THREAD_POOL; - private final int smallBufferSize; private final long serverDefaultsValidityPeriod; /** @@ -326,7 +326,6 @@ public DFSClient(URI nameNodeUri, ClientProtocol rpcNamenode, this.stats = stats; this.socketFactory = NetUtils.getSocketFactory(conf, ClientProtocol.class); this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf); - this.smallBufferSize = DFSUtilClient.getSmallBufferSize(conf); this.dtpReplaceDatanodeOnFailureReplication = (short) conf .getInt(HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure. MIN_REPLICATION, @@ -1242,7 +1241,7 @@ public DFSOutputStream create(String src, FsPermission permission, /** * Same as {@link #create(String, FsPermission, EnumSet, boolean, short, long, - * addition of Progressable, int, ChecksumOpt, InetSocketAddress[], String)} + * Progressable, int, ChecksumOpt, InetSocketAddress[], String)} * with the storagePolicy that is used to specify a specific storage policy * instead of inheriting any policy from this new file's parent directory. * This policy will be persisted in HDFS. A value of null means inheriting @@ -2001,8 +2000,17 @@ private long getStateByIndex(int stateIndex) throws IOException { * @see ClientProtocol#getStats() */ public FsStatus getDiskStatus() throws IOException { - return new FsStatus(getStateByIndex(0), - getStateByIndex(1), getStateByIndex(2)); + try (TraceScope ignored = tracer.newScope("getStats")) { + long[] states = namenode.getStats(); + return new FsStatus(getStateAtIndex(states, 0), + getStateAtIndex(states, 1), getStateAtIndex(states, 2)); + } catch (RemoteException re) { + throw re.unwrapRemoteException(); + } + } + + private long getStateAtIndex(long[] states, int index) { + return states.length > index ? states[index] : -1; } /** @@ -2190,6 +2198,24 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() } } + /** + * Get listing of all the snapshots for a snapshottable directory. + * + * @return Information about all the snapshots for a snapshottable directory + * @throws IOException If an I/O error occurred + * @see ClientProtocol#getSnapshotListing(String) + */ + public SnapshotStatus[] getSnapshotListing(String snapshotRoot) + throws IOException { + checkOpen(); + try (TraceScope ignored = tracer.newScope("getSnapshotListing")) { + return namenode.getSnapshotListing(snapshotRoot); + } catch (RemoteException re) { + throw re.unwrapRemoteException(); + } + } + + /** * Allow snapshot on a directory. * @@ -3132,6 +3158,35 @@ boolean isHDFSEncryptionEnabled() throws IOException { return getKeyProviderUri() != null; } + boolean isSnapshotTrashRootEnabled() throws IOException { + return getServerDefaults().getSnapshotTrashRootEnabled(); + } + + /** + * Get the snapshot root of a given file or directory if it exists. + * e.g. if /snapdir1 is a snapshottable directory and path given is + * /snapdir1/path/to/file, this method would return /snapdir1 + * @param path Path to a file or a directory. + * @return Not null if found in a snapshot root directory. + * @throws IOException + */ + String getSnapshotRoot(Path path) throws IOException { + SnapshottableDirectoryStatus[] dirStatusList = getSnapshottableDirListing(); + if (dirStatusList == null) { + return null; + } + for (SnapshottableDirectoryStatus dirStatus : dirStatusList) { + String currDir = dirStatus.getFullPath().toString(); + if (!currDir.endsWith(Path.SEPARATOR)) { + currDir += Path.SEPARATOR; + } + if (path.toUri().getPath().startsWith(currDir)) { + return currDir; + } + } + return null; + } + /** * Returns the SaslDataTransferClient configured for this DFSClient. * diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java index d36c0581c2153..3e4d1df8a5ca5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java @@ -19,7 +19,7 @@ import java.util.concurrent.atomic.AtomicLong; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 0676cf91a7d9e..4e21133d7ef93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -88,7 +88,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import javax.annotation.Nonnull; @@ -181,10 +181,13 @@ private boolean isPeriodicRefreshEnabled() { private byte[] oneByteBuf; // used for 'int read()' protected void addToLocalDeadNodes(DatanodeInfo dnInfo) { + DFSClient.LOG.debug("Add {} to local dead nodes, previously was {}.", + dnInfo, deadNodes); deadNodes.put(dnInfo, dnInfo); } protected void removeFromLocalDeadNodes(DatanodeInfo dnInfo) { + DFSClient.LOG.debug("Remove {} from local dead nodes.", dnInfo); deadNodes.remove(dnInfo); } @@ -1086,7 +1089,9 @@ protected DNAddrPair getBestNodeDNAddrPair(LocatedBlock block, final String dnAddr = chosenNode.getXferAddr(dfsClient.getConf().isConnectToDnViaHostname()); DFSClient.LOG.debug("Connecting to datanode {}", dnAddr); - InetSocketAddress targetAddr = NetUtils.createSocketAddr(dnAddr); + boolean uriCacheEnabled = dfsClient.getConf().isUriCacheEnabled(); + InetSocketAddress targetAddr = NetUtils.createSocketAddr(dnAddr, + -1, null, uriCacheEnabled); return new DNAddrPair(chosenNode, targetAddr, storageType, block); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java index 2113ae5c63544..fdd0072905fd4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java @@ -111,6 +111,7 @@ public enum OpType { SET_XATTR("op_set_xattr"), GET_SNAPSHOT_DIFF("op_get_snapshot_diff"), GET_SNAPSHOTTABLE_DIRECTORY_LIST("op_get_snapshottable_directory_list"), + GET_SNAPSHOT_LIST("op_get_snapshot_list"), TRUNCATE(CommonStatisticNames.OP_TRUNCATE), UNSET_EC_POLICY("op_unset_ec_policy"), UNSET_STORAGE_POLICY("op_unset_storage_policy"); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index a9e44cd5c0def..b9cbef07e1264 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -72,9 +72,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write.RECOVER_LEASE_ON_CLOSE_EXCEPTION_DEFAULT; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write.RECOVER_LEASE_ON_CLOSE_EXCEPTION_KEY; /**************************************************************** * DFSOutputStream creates files from a stream of bytes. @@ -126,6 +128,7 @@ public class DFSOutputStream extends FSOutputSummer protected final AtomicReference cachingStrategy; private FileEncryptionInfo fileEncryptionInfo; private int writePacketSize; + private boolean leaseRecovered = false; /** Use {@link ByteArrayManager} to create buffer for non-heartbeat packets.*/ protected DFSPacket createPacket(int packetSize, int chunksPerPkt, @@ -861,7 +864,14 @@ public void close() throws IOException { } protected synchronized void closeImpl() throws IOException { + boolean recoverLeaseOnCloseException = dfsClient.getConfiguration() + .getBoolean(RECOVER_LEASE_ON_CLOSE_EXCEPTION_KEY, + RECOVER_LEASE_ON_CLOSE_EXCEPTION_DEFAULT); if (isClosed()) { + if (!leaseRecovered) { + recoverLease(recoverLeaseOnCloseException); + } + LOG.debug("Closing an already closed stream. [Stream:{}, streamer:{}]", closed, getStreamer().streamerClosed()); try { @@ -896,6 +906,9 @@ protected synchronized void closeImpl() throws IOException { } completeFile(); } catch (ClosedChannelException ignored) { + } catch (IOException ioe) { + recoverLease(recoverLeaseOnCloseException); + throw ioe; } finally { // Failures may happen when flushing data. // Streamers may keep waiting for the new block information. @@ -906,7 +919,23 @@ protected synchronized void closeImpl() throws IOException { } } - private void completeFile() throws IOException { + /** + * If recoverLeaseOnCloseException is true and an exception occurs when + * closing a file, recover lease. + */ + private void recoverLease(boolean recoverLeaseOnCloseException) { + if (recoverLeaseOnCloseException) { + try { + dfsClient.endFileLease(fileId); + dfsClient.recoverLease(src); + leaseRecovered = true; + } catch (Exception e) { + LOG.warn("Fail to recover lease for {}", src, e); + } + } + } + + void completeFile() throws IOException { // get last block before destroying the streamer ExtendedBlock lastBlock = getStreamer().getBlock(); try (TraceScope ignored = @@ -1076,6 +1105,11 @@ public String toString() { return getClass().getSimpleName() + ":" + streamer; } + @VisibleForTesting + boolean isLeaseRecovered() { + return leaseRecovered; + } + static LocatedBlock addBlock(DatanodeInfo[] excludedNodes, DFSClient dfsClient, String src, ExtendedBlock prevBlock, long fileId, String[] favoredNodes, EnumSet allocFlags) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java index fa1cf34008ffb..9b27033bccdcb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.ReadOption; import org.apache.hadoop.hdfs.protocol.BlockType; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index 4222478f976de..7c3965647c54b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.CreateFlag; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java index 95aad12d928b2..55c64dc130d91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hdfs; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; -import com.google.common.primitives.SignedBytes; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.primitives.SignedBytes; import java.net.URISyntaxException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -1040,4 +1040,16 @@ public static String getEZTrashRoot(EncryptionZone ez, return (ezpath.equals("/") ? ezpath : ezpath + Path.SEPARATOR) + FileSystem.TRASH_PREFIX + Path.SEPARATOR + ugi.getShortUserName(); } + + /** + * Returns trash root in a snapshottable directory. + * @param ssRoot String of path to a snapshottable directory root. + * @param ugi user of trash owner. + * @return unqualified path of trash root. + */ + public static String getSnapshotTrashRoot(String ssRoot, + UserGroupInformation ugi) { + return (ssRoot.equals("/") ? ssRoot : ssRoot + Path.SEPARATOR) + + FileSystem.TRASH_PREFIX + Path.SEPARATOR + ugi.getShortUserName(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java index 1344c3b3a6ef6..d9daa37e201a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java @@ -42,7 +42,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.BlockWrite; @@ -80,11 +80,11 @@ import org.apache.htrace.core.TraceScope; import org.apache.htrace.core.Tracer; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; +import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; +import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalListener; +import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalNotification; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java index a573e8a22aad3..fd8263f88ed93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -294,7 +294,7 @@ void startProbeScheduler() { } /** - * Prode datanode by probe byte. + * Prode datanode by probe type. */ private void scheduleProbe(ProbeType type) { LOG.debug("Schedule probe datanode for probe type: {}.", type); @@ -376,9 +376,8 @@ public DatanodeLocalInfo call() throws Exception { } catch (Exception e) { LOG.error("Probe failed, datanode: {}, type: {}.", datanodeInfo, type, e); + deadNodeDetector.probeCallBack(this, false); } - - deadNodeDetector.probeCallBack(this, false); } } @@ -402,7 +401,7 @@ private void probeCallBack(Probe probe, boolean success) { } } else { if (probe.getType() == ProbeType.CHECK_SUSPECT) { - LOG.info("Add the node to dead node list: {}.", + LOG.warn("Probe failed, add suspect node to dead node list: {}.", probe.getDatanodeInfo()); addToDead(probe.getDatanodeInfo()); } @@ -415,11 +414,12 @@ private void probeCallBack(Probe probe, boolean success) { private void checkDeadNodes() { Set datanodeInfos = clearAndGetDetectedDeadNodes(); for (DatanodeInfo datanodeInfo : datanodeInfos) { - LOG.debug("Add dead node to check: {}.", datanodeInfo); if (!deadNodesProbeQueue.offer(datanodeInfo)) { LOG.debug("Skip to add dead node {} to check " + "since the probe queue is full.", datanodeInfo); break; + } else { + LOG.debug("Add dead node to check: {}.", datanodeInfo); } } state = State.IDLE; @@ -475,6 +475,7 @@ public synchronized void addNodeToDetect(DFSInputStream dfsInputStream, datanodeInfos.add(datanodeInfo); } + LOG.debug("Add datanode {} to suspectAndDeadNodes.", datanodeInfo); addSuspectNodeToDetect(datanodeInfo); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 450862b777078..076dbdd1767d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -19,9 +19,10 @@ package org.apache.hadoop.hdfs; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.commons.collections.list.TreeList; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; @@ -109,6 +110,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry; import org.apache.hadoop.hdfs.client.impl.SnapshotDiffReportGenerator; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; @@ -128,10 +130,12 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; +import java.util.Set; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; @@ -183,7 +187,7 @@ public void initialize(URI uri, Configuration conf) throws IOException { throw new IOException("Incomplete HDFS URI, no host: "+ uri); } - this.dfs = new DFSClient(uri, conf, statistics); + initDFSClient(uri, conf); this.uri = URI.create(uri.getScheme()+"://"+uri.getAuthority()); this.workingDir = getHomeDirectory(); @@ -197,6 +201,10 @@ public StorageStatistics provide() { }); } + void initDFSClient(URI theUri, Configuration conf) throws IOException { + this.dfs = new DFSClient(theUri, conf, statistics); + } + @Override public Path getWorkingDirectory() { return workingDir; @@ -1507,10 +1515,14 @@ protected boolean primitiveMkdir(Path f, FsPermission absolutePermission) @Override public void close() throws IOException { try { - dfs.closeOutputStreams(false); + if (dfs != null) { + dfs.closeOutputStreams(false); + } super.close(); } finally { - dfs.close(); + if (dfs != null) { + dfs.close(); + } } } @@ -1756,6 +1768,18 @@ public FileStatus next(final FileSystem fs, final Path p) }.resolve(this, absF); } + /** + * Synchronize client metadata state with Active NameNode. + *

+ * In HA the client synchronizes its state with the Active NameNode + * in order to guarantee subsequent read consistency from Observer Nodes. + * @throws IOException + */ + @Override + public void msync() throws IOException { + dfs.msync(); + } + @SuppressWarnings("deprecation") @Override public void createSymlink(final Path target, final Path link, @@ -2021,6 +2045,19 @@ public boolean isInSafeMode() throws IOException { return setSafeMode(SafeModeAction.SAFEMODE_GET, true); } + /** + * HDFS only. + * + * Returns if the NameNode enabled the snapshot trash root configuration + * dfs.namenode.snapshot.trashroot.enabled + * @return true if NameNode enabled snapshot trash root + * @throws IOException + * when there is an issue communicating with the NameNode + */ + public boolean isSnapshotTrashRootEnabled() throws IOException { + return dfs.isSnapshotTrashRootEnabled(); + } + /** @see org.apache.hadoop.hdfs.client.HdfsAdmin#allowSnapshot(Path) */ public void allowSnapshot(final Path path) throws IOException { statistics.incrementWriteOps(1); @@ -2057,6 +2094,7 @@ public void disallowSnapshot(final Path path) throws IOException { new FileSystemLinkResolver() { @Override public Void doCall(final Path p) throws IOException { + checkTrashRootAndRemoveIfEmpty(p); dfs.disallowSnapshot(getPathName(p)); return null; } @@ -2066,6 +2104,7 @@ public Void next(final FileSystem fs, final Path p) throws IOException { if (fs instanceof DistributedFileSystem) { DistributedFileSystem myDfs = (DistributedFileSystem)fs; + myDfs.checkTrashRootAndRemoveIfEmpty(p); myDfs.disallowSnapshot(p); } else { throw new UnsupportedOperationException("Cannot perform snapshot" @@ -2077,6 +2116,41 @@ public Void next(final FileSystem fs, final Path p) }.resolve(this, absF); } + /** + * Helper function to check if a trash root exists in the given directory, + * remove the trash root if it is empty, or throw IOException if not empty + * @param p Path to a directory. + */ + private void checkTrashRootAndRemoveIfEmpty(final Path p) throws IOException { + Path trashRoot = new Path(p, FileSystem.TRASH_PREFIX); + try { + // listStatus has 4 possible outcomes here: + // 1) throws FileNotFoundException: the trash root doesn't exist. + // 2) returns empty array: the trash path is an empty directory. + // 3) returns non-empty array, len >= 2: the trash root is not empty. + // 4) returns non-empty array, len == 1: + // i) if the element's path is exactly p, the trash path is not a dir. + // e.g. a file named .Trash. Ignore. + // ii) if the element's path isn't p, the trash root is not empty. + FileStatus[] fileStatuses = listStatus(trashRoot); + if (fileStatuses.length == 0) { + DFSClient.LOG.debug("Removing empty trash root {}", trashRoot); + delete(trashRoot, false); + } else { + if (fileStatuses.length == 1 + && !fileStatuses[0].isDirectory() + && !fileStatuses[0].getPath().equals(p)) { + // Ignore the trash path because it is not a directory. + DFSClient.LOG.warn("{} is not a directory.", trashRoot); + } else { + throw new IOException("Found non-empty trash root at " + + trashRoot + ". Rename or delete it, then try again."); + } + } + } catch (FileNotFoundException | AccessControlException ignored) { + } + } + @Override public Path createSnapshot(final Path path, final String snapshotName) throws IOException { @@ -2148,6 +2222,19 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() return dfs.getSnapshottableDirListing(); } + /** + * @return all the snapshots for a snapshottable directory + * @throws IOException + */ + public SnapshotStatus[] getSnapshotListing(Path snapshotRoot) + throws IOException { + Path absF = fixRelativePart(snapshotRoot); + statistics.incrementReadOps(1); + storageStatistics + .incrementOpCounter(OpType.GET_SNAPSHOT_LIST); + return dfs.getSnapshotListing(getPathName(absF)); + } + @Override public void deleteSnapshot(final Path snapshotDir, final String snapshotName) throws IOException { @@ -2871,6 +2958,80 @@ private void provisionEZTrash(String path, FsPermission trashPermission) setPermission(trashPath, trashPermission); } + /** + * HDFS only. + * + * Provision snapshottable directory trash. + * @param path Path to a snapshottable directory. + * @param trashPermission Expected FsPermission of the trash root. + * @return Path of the provisioned trash root + */ + public Path provisionSnapshotTrash(final Path path, + final FsPermission trashPermission) throws IOException { + Path absF = fixRelativePart(path); + return new FileSystemLinkResolver() { + @Override + public Path doCall(Path p) throws IOException { + return provisionSnapshotTrash(getPathName(p), trashPermission); + } + + @Override + public Path next(FileSystem fs, Path p) throws IOException { + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem myDfs = (DistributedFileSystem)fs; + return myDfs.provisionSnapshotTrash(p, trashPermission); + } + throw new UnsupportedOperationException( + "Cannot provisionSnapshotTrash through a symlink to" + + " a non-DistributedFileSystem: " + fs + " -> " + p); + } + }.resolve(this, absF); + } + + private Path provisionSnapshotTrash( + String pathStr, FsPermission trashPermission) throws IOException { + Path path = new Path(pathStr); + // Given path must be a snapshottable directory + FileStatus fileStatus = getFileStatus(path); + if (!fileStatus.isSnapshotEnabled()) { + throw new IllegalArgumentException( + path + " is not a snapshottable directory."); + } + + // Check if trash root already exists + Path trashPath = new Path(path, FileSystem.TRASH_PREFIX); + try { + FileStatus trashFileStatus = getFileStatus(trashPath); + String errMessage = "Can't provision trash for snapshottable directory " + + pathStr + " because trash path " + trashPath.toString() + + " already exists."; + if (!trashFileStatus.isDirectory()) { + errMessage += "\r\n" + + "WARNING: " + trashPath.toString() + " is not a directory."; + } + if (!trashFileStatus.getPermission().equals(trashPermission)) { + errMessage += "\r\n" + + "WARNING: Permission of " + trashPath.toString() + + " differs from provided permission " + trashPermission; + } + throw new FileAlreadyExistsException(errMessage); + } catch (FileNotFoundException ignored) { + // Trash path doesn't exist. Continue + } + + // Create trash root and set the permission + mkdir(trashPath, trashPermission); + setPermission(trashPath, trashPermission); + + // Print a warning if snapshot trash root feature is not enabled + if (!isSnapshotTrashRootEnabled()) { + DFSClient.LOG.warn("New trash is provisioned, but the snapshot trash root" + + " feature is disabled. This new trash but won't be automatically" + + " utilized unless the feature is enabled on the NameNode."); + } + return trashPath; + } + @Override public void setXAttr(Path path, final String name, final byte[] value, final EnumSet flag) throws IOException { @@ -3094,7 +3255,7 @@ public Void next(FileSystem fs, Path p) throws IOException { } /** - * Get erasure coding policy information for the specified path + * Get erasure coding policy information for the specified path. * * @param path The path of the file or directory * @return Returns the policy information if file or directory on the path @@ -3259,8 +3420,11 @@ public ECTopologyVerifierResult getECTopologyResultForPolicies( /** * Get the root directory of Trash for a path in HDFS. * 1. File in encryption zone returns /ez1/.Trash/username - * 2. File not in encryption zone, or encountered exception when checking - * the encryption zone of the path, returns /users/username/.Trash + * 2. File in snapshottable directory returns /snapdir1/.Trash/username + * if dfs.namenode.snapshot.trashroot.enabled is set to true. + * 3. In other cases, or encountered exception when checking the encryption + * zone or when checking snapshot root of the path, returns + * /users/username/.Trash * Caller appends either Current or checkpoint timestamp for trash destination * @param path the trash root of the path to be determined. * @return trash root @@ -3269,41 +3433,89 @@ public ECTopologyVerifierResult getECTopologyResultForPolicies( public Path getTrashRoot(Path path) { statistics.incrementReadOps(1); storageStatistics.incrementOpCounter(OpType.GET_TRASH_ROOT); + if (path == null) { + return super.getTrashRoot(null); + } + + // Snapshottable directory trash root, not null if path is inside a + // snapshottable directory and isSnapshotTrashRootEnabled is true from NN. + String ssTrashRoot = null; try { - if ((path == null) || !dfs.isHDFSEncryptionEnabled()) { - return super.getTrashRoot(path); + if (dfs.isSnapshotTrashRootEnabled()) { + String ssRoot = dfs.getSnapshotRoot(path); + if (ssRoot != null) { + ssTrashRoot = DFSUtilClient.getSnapshotTrashRoot(ssRoot, dfs.ugi); + } + } + } catch (IOException ioe) { + DFSClient.LOG.warn("Exception while checking whether the path is in a " + + "snapshottable directory", ioe); + } + + try { + if (!dfs.isHDFSEncryptionEnabled()) { + if (ssTrashRoot == null) { + // the path is not in a snapshottable directory and EZ is not enabled + return super.getTrashRoot(path); + } else { + return this.makeQualified(new Path(ssTrashRoot)); + } } } catch (IOException ioe) { DFSClient.LOG.warn("Exception while checking whether encryption zone is " + "supported", ioe); } - String parentSrc = path.isRoot()? - path.toUri().getPath():path.getParent().toUri().getPath(); + // HDFS encryption is enabled on the cluster at this point, does not + // necessary mean the given path is in an EZ hence the check. + String parentSrc = path.isRoot() ? + path.toUri().getPath() : path.getParent().toUri().getPath(); + String ezTrashRoot = null; try { EncryptionZone ez = dfs.getEZForPath(parentSrc); if ((ez != null)) { - return this.makeQualified( - new Path(DFSUtilClient.getEZTrashRoot(ez, dfs.ugi))); + ezTrashRoot = DFSUtilClient.getEZTrashRoot(ez, dfs.ugi); } } catch (IOException e) { DFSClient.LOG.warn("Exception in checking the encryption zone for the " + "path " + parentSrc + ". " + e.getMessage()); } - return super.getTrashRoot(path); + + if (ssTrashRoot == null) { + if (ezTrashRoot == null) { + // The path is neither in a snapshottable directory nor in an EZ + return super.getTrashRoot(path); + } else { + return this.makeQualified(new Path(ezTrashRoot)); + } + } else { + if (ezTrashRoot == null) { + return this.makeQualified(new Path(ssTrashRoot)); + } else { + // The path is in EZ and in a snapshottable directory + return this.makeQualified(new Path( + ssTrashRoot.length() > ezTrashRoot.length() ? + ssTrashRoot : ezTrashRoot)); + } + } } /** * Get all the trash roots of HDFS for current user or for all the users. - * 1. File deleted from non-encryption zone /user/username/.Trash - * 2. File deleted from encryption zones + * 1. File deleted from encryption zones * e.g., ez1 rooted at /ez1 has its trash root at /ez1/.Trash/$USER + * 2. File deleted from snapshottable directories + * if dfs.namenode.snapshot.trashroot.enabled is set to true. + * e.g., snapshottable directory /snapdir1 has its trash root + * at /snapdir1/.Trash/$USER + * 3. File deleted from other directories + * /user/username/.Trash * @param allUsers return trashRoots of all users if true, used by emptier * @return trash roots of HDFS */ @Override public Collection getTrashRoots(boolean allUsers) { - List ret = new ArrayList<>(); + Set ret = new HashSet<>(); // Get normal trash roots ret.addAll(super.getTrashRoots(allUsers)); @@ -3334,6 +3546,39 @@ public Collection getTrashRoots(boolean allUsers) { } catch (IOException e){ DFSClient.LOG.warn("Cannot get all encrypted trash roots", e); } + + try { + // Get snapshottable directory trash roots + if (dfs.isSnapshotTrashRootEnabled()) { + SnapshottableDirectoryStatus[] lst = dfs.getSnapshottableDirListing(); + if (lst != null) { + for (SnapshottableDirectoryStatus dirStatus : lst) { + String ssDir = dirStatus.getFullPath().toString(); + Path ssTrashRoot = new Path(ssDir, FileSystem.TRASH_PREFIX); + if (!exists(ssTrashRoot)) { + continue; + } + if (allUsers) { + for (FileStatus candidate : listStatus(ssTrashRoot)) { + if (exists(candidate.getPath())) { + ret.add(candidate); + } + } + } else { + Path userTrash = new Path(DFSUtilClient.getSnapshotTrashRoot( + ssDir, dfs.ugi)); + try { + ret.add(getFileStatus(userTrash)); + } catch (FileNotFoundException ignored) { + } + } + } + } + } + } catch (IOException e) { + DFSClient.LOG.warn("Cannot get snapshot trash roots", e); + } + return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java index 17d20fe014d1b..29073a1b38059 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java @@ -28,11 +28,11 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.util.KMSUtil; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.cache.Cache; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalListener; +import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalNotification; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java index 68577aad82501..31bc2d97a8662 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java @@ -35,8 +35,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PeerCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PeerCache.java index 0580ed536da15..79f313398be0f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PeerCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PeerCache.java @@ -23,9 +23,9 @@ import java.util.List; import java.util.Map.Entry; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.LinkedListMultimap; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.LinkedListMultimap; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java index 65d2c906a952c..efadedb8f082f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.util.StripedBlockUtil; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java index b37501d2e2be7..8bc6951f593ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.util.StripedBlockUtil; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java index 96cbb73a3896d..69aad06363c6c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.fs.ChecksumException; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java index 09dd8cb48ca79..e90e66ace4988 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java @@ -33,7 +33,7 @@ import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Progressable; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * This class extends {@link DataStreamer} to support writing striped blocks diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java new file mode 100644 index 0000000000000..936c65bd4dda0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java @@ -0,0 +1,2323 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.HadoopIllegalArgumentException; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.BlockStoragePolicySpi; +import org.apache.hadoop.fs.CacheFlag; +import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileEncryptionInfo; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.fs.FsStatus; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Options; +import org.apache.hadoop.fs.PartialListing; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.PathHandle; +import org.apache.hadoop.fs.QuotaUsage; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.fs.XAttrSetFlag; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.AclStatus; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.viewfs.ViewFileSystem; +import org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme; +import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; +import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; +import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; +import org.apache.hadoop.hdfs.protocol.CachePoolEntry; +import org.apache.hadoop.hdfs.protocol.CachePoolInfo; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; +import org.apache.hadoop.hdfs.protocol.EncryptionZone; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.HdfsPathHandle; +import org.apache.hadoop.hdfs.protocol.OpenFileEntry; +import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; +import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; +import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; +import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.io.MultipleIOException; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.token.DelegationTokenIssuer; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.Progressable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.IOException; + +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; + +/** + * The ViewDistributedFileSystem is an extended class to DistributedFileSystem + * with additional mounting functionality. The goal is to have better API + * compatibility for HDFS users when using mounting + * filesystem(ViewFileSystemOverloadScheme). + * The ViewFileSystemOverloadScheme{@link ViewFileSystemOverloadScheme} is a new + * filesystem with inherited mounting functionality from ViewFileSystem. + * For the user who is using ViewFileSystemOverloadScheme by setting + * fs.hdfs.impl=org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme, now + * they can set fs.hdfs.impl=org.apache.hadoop.hdfs.ViewDistributedFileSystem. + * So, that the hdfs users will get closely compatible API with mount + * functionality. For the rest of all other schemes can continue to use + * ViewFileSystemOverloadScheme class directly for mount functionality. Please + * note that ViewFileSystemOverloadScheme provides only + * ViewFileSystem{@link ViewFileSystem} APIs. + * If user configured this class but no mount point configured? Then it will + * simply work as existing DistributedFileSystem class. If user configured both + * fs.hdfs.impl to this class and mount configurations, then users will be able + * to make calls the APIs available in this class, they are nothing but DFS + * APIs, but they will be delegated to viewfs functionality. Please note, APIs + * without any path in arguments( ex: isInSafeMode), will be delegated to + * default filesystem only, that is the configured fallback link. If you want to + * make these API calls on specific child filesystem, you may want to initialize + * them separately and call. In ViewDistributedFileSystem, we strongly recommend + * to configure linkFallBack when you add mount links and it's recommended to + * point be to your base cluster, usually your current fs.defaultFS if that's + * pointing to hdfs. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class ViewDistributedFileSystem extends DistributedFileSystem { + private static final Logger LOGGER = + LoggerFactory.getLogger(ViewDistributedFileSystem.class); + + // A mounting file system. + private ViewFileSystemOverloadScheme vfs; + // A default DFS, which should have set via linkFallback + private DistributedFileSystem defaultDFS; + + @Override + public void initialize(URI uri, Configuration conf) throws IOException { + super.initialize(uri, conf); + try { + this.vfs = tryInitializeMountingViewFs(uri, conf); + } catch (IOException ioe) { + LOGGER.debug(new StringBuilder("Mount tree initialization failed with ") + .append("the reason => {}. Falling back to regular DFS") + .append(" initialization. Please re-initialize the fs after updating") + .append(" mount point.").toString(), ioe.getMessage()); + // Previous super.initialize would have skipped the dfsclient init and + // setWorkingDirectory as we planned to initialize vfs. Since vfs init + // failed, let's init dfsClient now. + super.initDFSClient(uri, conf); + super.setWorkingDirectory(super.getHomeDirectory()); + return; + } + + setConf(conf); + // A child DFS with the current initialized URI. This must be same as + // fallback fs. The fallback must point to root of your filesystems. + // Some APIs(without path in argument, for example isInSafeMode) will + // support only for base cluster filesystem. Only that APIs will use this + // fs. + defaultDFS = (DistributedFileSystem) this.vfs.getFallbackFileSystem(); + // Please don't access internal dfs client directly except in tests. + dfs = (defaultDFS != null) ? defaultDFS.dfs : null; + super.setWorkingDirectory(this.vfs.getHomeDirectory()); + } + + @Override + void initDFSClient(URI uri, Configuration conf) throws IOException { + // Since we plan to initialize vfs in this class, we will not need to + // initialize DFS client. + } + + public ViewDistributedFileSystem() { + } + + private ViewFileSystemOverloadScheme tryInitializeMountingViewFs(URI theUri, + Configuration conf) throws IOException { + ViewFileSystemOverloadScheme viewFs = new ViewFileSystemOverloadScheme(); + viewFs.setSupportAutoAddingFallbackOnNoMounts(false); + viewFs.initialize(theUri, conf); + return viewFs; + } + + @Override + public URI getUri() { + if (this.vfs == null) { + return super.getUri(); + } + return this.vfs.getUri(); + } + + @Override + public String getScheme() { + if (this.vfs == null) { + return super.getScheme(); + } + return this.vfs.getScheme(); + } + + @Override + public Path getWorkingDirectory() { + if (this.vfs == null) { + return super.getWorkingDirectory(); + } + return this.vfs.getWorkingDirectory(); + } + + @Override + public void setWorkingDirectory(Path dir) { + if (this.vfs == null) { + super.setWorkingDirectory(dir); + return; + } + this.vfs.setWorkingDirectory(dir); + } + + @Override + public Path getHomeDirectory() { + if (super.dfs == null) { + return null; + } + if (this.vfs == null) { + return super.getHomeDirectory(); + } + return this.vfs.getHomeDirectory(); + } + + /** + * Returns only default cluster getHedgedReadMetrics. + */ + @Override + public DFSHedgedReadMetrics getHedgedReadMetrics() { + if (this.vfs == null) { + return super.getHedgedReadMetrics(); + } + checkDefaultDFS(defaultDFS, "getHedgedReadMetrics"); + return defaultDFS.getHedgedReadMetrics(); + } + + @Override + public BlockLocation[] getFileBlockLocations(FileStatus fs, long start, + long len) throws IOException { + if (this.vfs == null) { + return super.getFileBlockLocations(fs, start, len); + } + return this.vfs.getFileBlockLocations(fs, start, len); + } + + @Override + public BlockLocation[] getFileBlockLocations(Path p, final long start, + final long len) throws IOException { + if (this.vfs == null) { + return super.getFileBlockLocations(p, start, len); + } + return this.vfs.getFileBlockLocations(p, start, len); + } + + @Override + public void setVerifyChecksum(final boolean verifyChecksum) { + if (this.vfs == null) { + super.setVerifyChecksum(verifyChecksum); + return; + } + this.vfs.setVerifyChecksum(verifyChecksum); + } + + @Override + public boolean recoverLease(final Path f) throws IOException { + if (this.vfs == null) { + return super.recoverLease(f); + } + + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "recoverLease"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .recoverLease(mountPathInfo.getPathOnTarget()); + } + + @Override + public FSDataInputStream open(final Path f, final int bufferSize) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + return super.open(f, bufferSize); + } + + return this.vfs.open(f, bufferSize); + } + + @Override + public FSDataInputStream open(PathHandle fd, int bufferSize) + throws IOException { + if (this.vfs == null) { + return super.open(fd, bufferSize); + } + return this.vfs.open(fd, bufferSize); + } + + @Override + protected HdfsPathHandle createPathHandle(FileStatus st, + Options.HandleOpt... opts) { + if (this.vfs == null) { + return super.createPathHandle(st, opts); + } + throw new UnsupportedOperationException(); + } + + @Override + public FSDataOutputStream append(final Path f, final int bufferSize, + final Progressable progress) throws IOException { + if (this.vfs == null) { + return super.append(f, bufferSize, progress); + } + return this.vfs.append(f, bufferSize, progress); + } + + @Override + public FSDataOutputStream append(Path f, final EnumSet flag, + final int bufferSize, final Progressable progress) throws IOException { + if (this.vfs == null) { + return super.append(f, flag, bufferSize, progress); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "append"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .append(mountPathInfo.getPathOnTarget(), flag, bufferSize, progress); + } + + @Override + public FSDataOutputStream append(Path f, final EnumSet flag, + final int bufferSize, final Progressable progress, + final InetSocketAddress[] favoredNodes) throws IOException { + if (this.vfs == null) { + return super.append(f, flag, bufferSize, progress, favoredNodes); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "append"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .append(mountPathInfo.getPathOnTarget(), flag, bufferSize, progress, + favoredNodes); + } + + @Override + public FSDataOutputStream create(Path f, FsPermission permission, + boolean overwrite, int bufferSize, short replication, long blockSize, + Progressable progress) throws IOException { + if (this.vfs == null) { + return super + .create(f, permission, overwrite, bufferSize, replication, blockSize, + progress); + } + return this.vfs + .create(f, permission, overwrite, bufferSize, replication, blockSize, + progress); + } + + @Override + public HdfsDataOutputStream create(final Path f, + final FsPermission permission, final boolean overwrite, + final int bufferSize, final short replication, final long blockSize, + final Progressable progress, final InetSocketAddress[] favoredNodes) + throws IOException { + if (this.vfs == null) { + return super + .create(f, permission, overwrite, bufferSize, replication, blockSize, + progress, favoredNodes); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "create"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .create(mountPathInfo.getPathOnTarget(), permission, overwrite, + bufferSize, replication, blockSize, progress, favoredNodes); + } + + @Override + public FSDataOutputStream create(final Path f, final FsPermission permission, + final EnumSet cflags, final int bufferSize, + final short replication, final long blockSize, + final Progressable progress, final Options.ChecksumOpt checksumOpt) + throws IOException { + if (this.vfs == null) { + return super + .create(f, permission, cflags, bufferSize, replication, blockSize, + progress, checksumOpt); + } + return vfs.create(f, permission, cflags, bufferSize, replication, blockSize, + progress, checksumOpt); + } + + void checkDFS(FileSystem fs, String methodName) { + if (!(fs instanceof DistributedFileSystem)) { + String msg = new StringBuilder("This API:").append(methodName) + .append(" is specific to DFS. Can't run on other fs:") + .append(fs.getUri()).toString(); + throw new UnsupportedOperationException(msg); + } + } + + void checkDefaultDFS(FileSystem fs, String methodName) { + if (fs == null) { + String msg = new StringBuilder("This API:").append(methodName).append( + " cannot be supported without default cluster(that is linkFallBack).") + .toString(); + throw new UnsupportedOperationException(msg); + } + } + + @Override + // DFS specific API + protected HdfsDataOutputStream primitiveCreate(Path f, + FsPermission absolutePermission, EnumSet flag, int bufferSize, + short replication, long blockSize, Progressable progress, + Options.ChecksumOpt checksumOpt) throws IOException { + if (this.vfs == null) { + return super + .primitiveCreate(f, absolutePermission, flag, bufferSize, replication, + blockSize, progress, checksumOpt); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "primitiveCreate"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .primitiveCreate(f, absolutePermission, flag, bufferSize, replication, + blockSize, progress, checksumOpt); + } + + @Override + public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, + EnumSet flags, int bufferSize, short replication, + long blockSize, Progressable progress) throws IOException { + if (this.vfs == null) { + return super + .createNonRecursive(f, permission, flags, bufferSize, replication, + bufferSize, progress); + } + return this.vfs + .createNonRecursive(f, permission, flags, bufferSize, replication, + bufferSize, progress); + } + + @Override + public boolean setReplication(final Path f, final short replication) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + return super.setReplication(f, replication); + } + return this.vfs.setReplication(f, replication); + } + + @Override + public void setStoragePolicy(Path src, String policyName) throws IOException { + if (this.vfs == null) { + super.setStoragePolicy(src, policyName); + return; + } + this.vfs.setStoragePolicy(src, policyName); + } + + @Override + public void unsetStoragePolicy(Path src) throws IOException { + if (this.vfs == null) { + super.unsetStoragePolicy(src); + return; + } + this.vfs.unsetStoragePolicy(src); + } + + @Override + public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException { + if (this.vfs == null) { + return super.getStoragePolicy(src); + } + return this.vfs.getStoragePolicy(src); + } + + @Override + public Collection getAllStoragePolicies() + throws IOException { + if (this.vfs == null) { + return super.getAllStoragePolicies(); + } + Collection allStoragePolicies = + this.vfs.getAllStoragePolicies(); + return (Collection) allStoragePolicies; + } + + @Override + public long getBytesWithFutureGenerationStamps() throws IOException { + if (this.vfs == null) { + return super.getBytesWithFutureGenerationStamps(); + } + checkDefaultDFS(defaultDFS, "getBytesWithFutureGenerationStamps"); + return defaultDFS.getBytesWithFutureGenerationStamps(); + } + + @Deprecated + @Override + public BlockStoragePolicy[] getStoragePolicies() throws IOException { + if (this.vfs == null) { + return super.getStoragePolicies(); + } + checkDefaultDFS(defaultDFS, "getStoragePolicies"); + return defaultDFS.getStoragePolicies(); + } + + @Override + //Make sure your target fs supports this API, otherwise you will get + // Unsupported operation exception. + public void concat(Path trg, Path[] psrcs) throws IOException { + if (this.vfs == null) { + super.concat(trg, psrcs); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(trg, getConf()); + mountPathInfo.getTargetFs().concat(mountPathInfo.getPathOnTarget(), psrcs); + } + + @SuppressWarnings("deprecation") + @Override + public boolean rename(final Path src, final Path dst) throws IOException { + if (this.vfs == null) { + return super.rename(src, dst); + } + return this.vfs.rename(src, dst); + } + + @SuppressWarnings("deprecation") + @Override + public void rename(Path src, Path dst, final Options.Rename... options) + throws IOException { + if (this.vfs == null) { + super.rename(src, dst, options); + return; + } + + // TODO: revisit + ViewFileSystemOverloadScheme.MountPathInfo mountSrcPathInfo = + this.vfs.getMountPathInfo(src, getConf()); + checkDFS(mountSrcPathInfo.getTargetFs(), "rename"); + + ViewFileSystemOverloadScheme.MountPathInfo mountDstPathInfo = + this.vfs.getMountPathInfo(src, getConf()); + checkDFS(mountDstPathInfo.getTargetFs(), "rename"); + + //Check both in same cluster. + if (!mountSrcPathInfo.getTargetFs().getUri() + .equals(mountDstPathInfo.getTargetFs().getUri())) { + throw new HadoopIllegalArgumentException( + "Can't rename across file systems."); + } + + ((DistributedFileSystem) mountSrcPathInfo.getTargetFs()) + .rename(mountSrcPathInfo.getPathOnTarget(), + mountDstPathInfo.getPathOnTarget(), options); + } + + @Override + public boolean truncate(final Path f, final long newLength) + throws IOException { + if (this.vfs == null) { + return super.truncate(f, newLength); + } + return this.vfs.truncate(f, newLength); + } + + public boolean delete(final Path f, final boolean recursive) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + return super.delete(f, recursive); + } + return this.vfs.delete(f, recursive); + } + + @Override + public ContentSummary getContentSummary(Path f) throws IOException { + if (this.vfs == null) { + return super.getContentSummary(f); + } + return this.vfs.getContentSummary(f); + } + + @Override + public QuotaUsage getQuotaUsage(Path f) throws IOException { + if (this.vfs == null) { + return super.getQuotaUsage(f); + } + return this.vfs.getQuotaUsage(f); + } + + @Override + public void setQuota(Path src, final long namespaceQuota, + final long storagespaceQuota) throws IOException { + if (this.vfs == null) { + super.setQuota(src, namespaceQuota, storagespaceQuota); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(src, getConf()); + mountPathInfo.getTargetFs() + .setQuota(mountPathInfo.getPathOnTarget(), namespaceQuota, + storagespaceQuota); + } + + @Override + public void setQuotaByStorageType(Path src, final StorageType type, + final long quota) throws IOException { + if (this.vfs == null) { + super.setQuotaByStorageType(src, type, quota); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(src, getConf()); + mountPathInfo.getTargetFs() + .setQuotaByStorageType(mountPathInfo.getPathOnTarget(), type, quota); + } + + @Override + public FileStatus[] listStatus(Path p) throws IOException { + if (this.vfs == null) { + return super.listStatus(p); + } + return this.vfs.listStatus(p); + } + + @Override + public RemoteIterator listLocatedStatus(final Path f, + final PathFilter filter) throws FileNotFoundException, IOException { + if (this.vfs == null) { + return super.listLocatedStatus(f, filter); + } + return this.vfs.listLocatedStatus(f, filter); + } + + @Override + public RemoteIterator listStatusIterator(final Path p) + throws IOException { + if (this.vfs == null) { + return super.listStatusIterator(p); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(p, getConf()); + return mountPathInfo.getTargetFs() + .listStatusIterator(mountPathInfo.getPathOnTarget()); + } + + @Override + public RemoteIterator> batchedListStatusIterator( + final List paths) throws IOException { + if (this.vfs == null) { + return super.batchedListStatusIterator(paths); + } + // TODO: revisit for correct implementation. + return this.defaultDFS.batchedListStatusIterator(paths); + } + + @Override + public RemoteIterator> batchedListLocatedStatusIterator( + final List paths) throws IOException { + if (this.vfs == null) { + return super.batchedListLocatedStatusIterator(paths); + } + // TODO: revisit for correct implementation. + return this.defaultDFS.batchedListLocatedStatusIterator(paths); + } + + public boolean mkdir(Path f, FsPermission permission) throws IOException { + if (this.vfs == null) { + return super.mkdir(f, permission); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "mkdir"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .mkdir(mountPathInfo.getPathOnTarget(), permission); + } + + @Override + public boolean mkdirs(Path f, FsPermission permission) throws IOException { + if (this.vfs == null) { + return super.mkdirs(f, permission); + } + return this.vfs.mkdirs(f, permission); + } + + @SuppressWarnings("deprecation") + @Override + protected boolean primitiveMkdir(Path f, FsPermission absolutePermission) + throws IOException { + if (this.vfs == null) { + return super.primitiveMkdir(f, absolutePermission); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "primitiveMkdir"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .primitiveMkdir(mountPathInfo.getPathOnTarget(), absolutePermission); + } + + @Override + public void close() throws IOException { + if (this.vfs != null) { + this.vfs.close(); + } + super.close(); + } + + @InterfaceAudience.Private + @Override + public DFSClient getClient() { + if (this.vfs == null) { + return super.getClient(); + } + checkDefaultDFS(defaultDFS, "getClient"); + return defaultDFS.getClient(); + } + + @Override + public FsStatus getStatus(Path p) throws IOException { + if (this.vfs == null) { + return super.getStatus(p); + } + return this.vfs.getStatus(p); + } + + @Override + public long getMissingBlocksCount() throws IOException { + if (this.vfs == null) { + return super.getMissingBlocksCount(); + } + checkDefaultDFS(defaultDFS, "getMissingBlocksCount"); + return defaultDFS.getMissingBlocksCount(); + } + + @Override + public long getPendingDeletionBlocksCount() throws IOException { + if (this.vfs == null) { + return super.getPendingDeletionBlocksCount(); + } + checkDefaultDFS(defaultDFS, "getPendingDeletionBlocksCount"); + return defaultDFS.getPendingDeletionBlocksCount(); + } + + @Override + public long getMissingReplOneBlocksCount() throws IOException { + if (this.vfs == null) { + return super.getMissingReplOneBlocksCount(); + } + checkDefaultDFS(defaultDFS, "getMissingReplOneBlocksCount"); + return defaultDFS.getMissingReplOneBlocksCount(); + } + + @Override + public long getLowRedundancyBlocksCount() throws IOException { + if (this.vfs == null) { + return super.getLowRedundancyBlocksCount(); + } + checkDefaultDFS(defaultDFS, "getLowRedundancyBlocksCount"); + return defaultDFS.getLowRedundancyBlocksCount(); + } + + @Override + public long getCorruptBlocksCount() throws IOException { + if (this.vfs == null) { + return super.getCorruptBlocksCount(); + } + checkDefaultDFS(defaultDFS, "getCorruptBlocksCount"); + return defaultDFS.getLowRedundancyBlocksCount(); + } + + @Override + public RemoteIterator listCorruptFileBlocks(final Path path) + throws IOException { + if (this.vfs == null) { + return super.listCorruptFileBlocks(path); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + return mountPathInfo.getTargetFs() + .listCorruptFileBlocks(mountPathInfo.getPathOnTarget()); + } + + @Override + public DatanodeInfo[] getDataNodeStats() throws IOException { + if (this.vfs == null) { + return super.getDataNodeStats(); + } + checkDefaultDFS(defaultDFS, "getDataNodeStats"); + return defaultDFS.getDataNodeStats(); + } + + @Override + public DatanodeInfo[] getDataNodeStats( + final HdfsConstants.DatanodeReportType type) throws IOException { + if (this.vfs == null) { + return super.getDataNodeStats(type); + } + checkDefaultDFS(defaultDFS, "getDataNodeStats"); + return defaultDFS.getDataNodeStats(type); + } + + @Override + public boolean setSafeMode(HdfsConstants.SafeModeAction action) + throws IOException { + if (this.vfs == null) { + return super.setSafeMode(action); + } + checkDefaultDFS(defaultDFS, "setSafeMode"); + return defaultDFS.setSafeMode(action); + } + + @Override + public boolean setSafeMode(HdfsConstants.SafeModeAction action, + boolean isChecked) throws IOException { + if (this.vfs == null) { + return super.setSafeMode(action, isChecked); + } + checkDefaultDFS(defaultDFS, "setSafeMode"); + return defaultDFS.setSafeMode(action, isChecked); + } + + @Override + public boolean saveNamespace(long timeWindow, long txGap) throws IOException { + if (this.vfs == null) { + return super.saveNamespace(timeWindow, txGap); + } + checkDefaultDFS(defaultDFS, "saveNamespace"); + return defaultDFS.saveNamespace(timeWindow, txGap); + } + + @Override + public void saveNamespace() throws IOException { + if (this.vfs == null) { + super.saveNamespace(); + return; + } + checkDefaultDFS(defaultDFS, "saveNamespace"); + defaultDFS.saveNamespace(); + } + + @Override + public long rollEdits() throws IOException { + if (this.vfs == null) { + return super.rollEdits(); + } + checkDefaultDFS(defaultDFS, "rollEdits"); + return defaultDFS.rollEdits(); + } + + @Override + public boolean restoreFailedStorage(String arg) throws IOException { + if (this.vfs == null) { + return super.restoreFailedStorage(arg); + } + checkDefaultDFS(defaultDFS, "restoreFailedStorage"); + return defaultDFS.restoreFailedStorage(arg); + } + + @Override + public void refreshNodes() throws IOException { + if (this.vfs == null) { + super.refreshNodes(); + return; + } + checkDefaultDFS(defaultDFS, "refreshNodes"); + defaultDFS.refreshNodes(); + } + + @Override + public void finalizeUpgrade() throws IOException { + if (this.vfs == null) { + super.finalizeUpgrade(); + return; + } + checkDefaultDFS(defaultDFS, "finalizeUpgrade"); + defaultDFS.finalizeUpgrade(); + } + + @Override + public boolean upgradeStatus() throws IOException { + if (this.vfs == null) { + return super.upgradeStatus(); + } + checkDefaultDFS(defaultDFS, "upgradeStatus"); + return defaultDFS.upgradeStatus(); + } + + @Override + public RollingUpgradeInfo rollingUpgrade( + HdfsConstants.RollingUpgradeAction action) throws IOException { + if (this.vfs == null) { + return super.rollingUpgrade(action); + } + checkDefaultDFS(defaultDFS, "rollingUpgrade"); + return defaultDFS.rollingUpgrade(action); + } + + @Override + public void metaSave(String pathname) throws IOException { + if (this.vfs == null) { + super.metaSave(pathname); + return; + } + checkDefaultDFS(defaultDFS, "metaSave"); + defaultDFS.metaSave(pathname); + } + + @Override + public FsServerDefaults getServerDefaults() throws IOException { + if (this.vfs == null) { + return super.getServerDefaults(); + } + checkDefaultDFS(defaultDFS, "getServerDefaults"); + //TODO: Need to revisit. + return defaultDFS.getServerDefaults(); + } + + @Override + public FileStatus getFileStatus(final Path f) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + return super.getFileStatus(f); + } + return this.vfs.getFileStatus(f); + } + + @SuppressWarnings("deprecation") + @Override + public void createSymlink(final Path target, final Path link, + final boolean createParent) throws IOException { + // Regular DFS behavior + if (this.vfs == null) { + super.createSymlink(target, link, createParent); + return; + } + + throw new UnsupportedOperationException( + "createSymlink is not supported in ViewHDFS"); + } + + @Override + public boolean supportsSymlinks() { + if (this.vfs == null) { + return super.supportsSymlinks(); + } + // we can enabled later if we want to support symlinks. + return false; + } + + @Override + public FileStatus getFileLinkStatus(final Path f) throws IOException { + if (this.vfs == null) { + return super.getFileLinkStatus(f); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + return mountPathInfo.getTargetFs() + .getFileLinkStatus(mountPathInfo.getPathOnTarget()); + } + + @Override + public Path getLinkTarget(Path path) throws IOException { + if(this.vfs==null){ + return super.getLinkTarget(path); + } + return this.vfs.getLinkTarget(path); + } + + @Override + protected Path resolveLink(Path f) throws IOException { + if(this.vfs==null){ + return super.resolveLink(f); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "resolveLink"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .resolveLink(mountPathInfo.getPathOnTarget()); + } + + @Override + public FileChecksum getFileChecksum(final Path f) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + return super.getFileChecksum(f); + } + return this.vfs.getFileChecksum(f); + } + + @Override + public void setPermission(final Path f, final FsPermission permission) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + super.setPermission(f, permission); + return; + } + this.vfs.setPermission(f, permission); + } + + @Override + public void setOwner(final Path f, final String username, + final String groupname) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + super.setOwner(f, username, groupname); + return; + } + this.vfs.setOwner(f, username, groupname); + } + + @Override + public void setTimes(final Path f, final long mtime, final long atime) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + super.setTimes(f, mtime, atime); + return; + } + this.vfs.setTimes(f, mtime, atime); + } + + @Override + // DFS specific API + protected int getDefaultPort() { + return super.getDefaultPort(); + } + + /** + * If no mount points configured, it works same as + * {@link DistributedFileSystem#getDelegationToken(String)}. If + * there are mount points configured and if default fs(linkFallback) + * configured, then it will return default fs delegation token. Otherwise + * it will return null. + */ + @Override + public Token getDelegationToken(String renewer) + throws IOException { + if (this.vfs == null) { + return super.getDelegationToken(renewer); + } + + if (defaultDFS != null) { + return defaultDFS.getDelegationToken(renewer); + } + return null; + } + + @Override + public void setBalancerBandwidth(long bandwidth) throws IOException { + if (this.vfs == null) { + super.setBalancerBandwidth(bandwidth); + return; + } + checkDefaultDFS(defaultDFS, "setBalancerBandwidth"); + defaultDFS.setBalancerBandwidth(bandwidth); + } + + @Override + public String getCanonicalServiceName() { + if (this.vfs == null) { + return super.getCanonicalServiceName(); + } + checkDefaultDFS(defaultDFS, "getCanonicalServiceName"); + return defaultDFS.getCanonicalServiceName(); + } + + @Override + protected URI canonicalizeUri(URI uri) { + if (this.vfs == null) { + return super.canonicalizeUri(uri); + } + + return vfs.canonicalizeUri(uri); + } + + @Override + public boolean isInSafeMode() throws IOException { + if (this.vfs == null) { + return super.isInSafeMode(); + } + checkDefaultDFS(defaultDFS, "isInSafeMode"); + return defaultDFS.isInSafeMode(); + } + + @Override + // DFS specific API + public void allowSnapshot(Path path) throws IOException { + if (this.vfs == null) { + super.allowSnapshot(path); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "allowSnapshot"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .allowSnapshot(mountPathInfo.getPathOnTarget()); + } + + @Override + public void disallowSnapshot(final Path path) throws IOException { + if (this.vfs == null) { + super.disallowSnapshot(path); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "disallowSnapshot"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .disallowSnapshot(mountPathInfo.getPathOnTarget()); + } + + @Override + public Path createSnapshot(Path path, String snapshotName) + throws IOException { + if (this.vfs == null) { + return super.createSnapshot(path, snapshotName); + } + return this.vfs.createSnapshot(path, snapshotName); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + if (this.vfs == null) { + super.renameSnapshot(path, snapshotOldName, snapshotNewName); + return; + } + this.vfs.renameSnapshot(path, snapshotOldName, snapshotNewName); + } + + @Override + //Ony for HDFS users + public SnapshottableDirectoryStatus[] getSnapshottableDirListing() + throws IOException { + if (this.vfs == null) { + return super.getSnapshottableDirListing(); + } + checkDefaultDFS(defaultDFS, "getSnapshottableDirListing"); + return defaultDFS.getSnapshottableDirListing(); + } + + @Override + public void deleteSnapshot(Path path, String snapshotName) + throws IOException { + if (this.vfs == null) { + super.deleteSnapshot(path, snapshotName); + return; + } + this.vfs.deleteSnapshot(path, snapshotName); + } + + @Override + public RemoteIterator snapshotDiffReportListingRemoteIterator( + final Path snapshotDir, final String fromSnapshot, + final String toSnapshot) throws IOException { + if (this.vfs == null) { + return super + .snapshotDiffReportListingRemoteIterator(snapshotDir, fromSnapshot, + toSnapshot); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(snapshotDir, getConf()); + checkDFS(mountPathInfo.getTargetFs(), + "snapshotDiffReportListingRemoteIterator"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .snapshotDiffReportListingRemoteIterator( + mountPathInfo.getPathOnTarget(), fromSnapshot, toSnapshot); + } + + @Override + public SnapshotDiffReport getSnapshotDiffReport(final Path snapshotDir, + final String fromSnapshot, final String toSnapshot) throws IOException { + if (this.vfs == null) { + return super.getSnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(snapshotDir, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "getSnapshotDiffReport"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .getSnapshotDiffReport(mountPathInfo.getPathOnTarget(), fromSnapshot, + toSnapshot); + } + + @Override + public boolean isFileClosed(final Path src) throws IOException { + if (this.vfs == null) { + return super.isFileClosed(src); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(src, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "isFileClosed"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .isFileClosed(mountPathInfo.getPathOnTarget()); + } + + @Override + public long addCacheDirective(CacheDirectiveInfo info) throws IOException { + if (this.vfs == null) { + return super.addCacheDirective(info); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(info.getPath(), getConf()); + checkDFS(mountPathInfo.getTargetFs(), "addCacheDirective"); + + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .addCacheDirective(new CacheDirectiveInfo.Builder(info) + .setPath(mountPathInfo.getPathOnTarget()).build()); + } + + @Override + public long addCacheDirective(CacheDirectiveInfo info, + EnumSet flags) throws IOException { + if (this.vfs == null) { + return super.addCacheDirective(info, flags); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(info.getPath(), getConf()); + checkDFS(mountPathInfo.getTargetFs(), "addCacheDirective"); + + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .addCacheDirective(new CacheDirectiveInfo.Builder(info) + .setPath(mountPathInfo.getPathOnTarget()).build(), flags); + } + + @Override + public void modifyCacheDirective(CacheDirectiveInfo info) throws IOException { + if (this.vfs == null) { + super.modifyCacheDirective(info); + return; + } + if (info.getPath() != null) { + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(info.getPath(), getConf()); + checkDFS(mountPathInfo.getTargetFs(), "modifyCacheDirective"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .modifyCacheDirective(new CacheDirectiveInfo.Builder(info) + .setPath(mountPathInfo.getPathOnTarget()).build()); + return; + } + + // No path available in CacheDirectiveInfo, Let's shoot to all child fs. + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.modifyCacheDirective(info); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public void modifyCacheDirective(CacheDirectiveInfo info, + EnumSet flags) throws IOException { + if (this.vfs == null) { + super.modifyCacheDirective(info, flags); + return; + } + if (info.getPath() != null) { + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(info.getPath(), getConf()); + checkDFS(mountPathInfo.getTargetFs(), "modifyCacheDirective"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .modifyCacheDirective(new CacheDirectiveInfo.Builder(info) + .setPath(mountPathInfo.getPathOnTarget()).build(), flags); + return; + } + // No path available in CacheDirectiveInfo, Let's shoot to all child fs. + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.modifyCacheDirective(info, flags); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public void removeCacheDirective(long id) throws IOException { + if (this.vfs == null) { + super.removeCacheDirective(id); + return; + } + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.removeCacheDirective(id); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public RemoteIterator listCacheDirectives( + CacheDirectiveInfo filter) throws IOException { + if (this.vfs == null) { + return super.listCacheDirectives(filter); + } + + if (filter != null && filter.getPath() != null) { + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(filter.getPath(), getConf()); + checkDFS(mountPathInfo.getTargetFs(), "listCacheDirectives"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .listCacheDirectives(new CacheDirectiveInfo.Builder(filter) + .setPath(mountPathInfo.getPathOnTarget()).build()); + } + + // No path available in filter. Let's try to shoot to all child fs. + final List> iters = new ArrayList<>(); + for (FileSystem fs : getChildFileSystems()) { + if (fs instanceof DistributedFileSystem) { + iters.add(((DistributedFileSystem) fs).listCacheDirectives(filter)); + } + } + if (iters.size() == 0) { + throw new UnsupportedOperationException( + "No DFS found in child fs. This API can't be supported in non DFS"); + } + + return new RemoteIterator() { + int currIdx = 0; + RemoteIterator currIter = iters.get(currIdx++); + + @Override + public boolean hasNext() throws IOException { + if (currIter.hasNext()) { + return true; + } + while (currIdx < iters.size()) { + currIter = iters.get(currIdx++); + if (currIter.hasNext()) { + return true; + } + } + return false; + } + + @Override + public CacheDirectiveEntry next() throws IOException { + if (hasNext()) { + return currIter.next(); + } + throw new NoSuchElementException("No more elements"); + } + }; + } + + //Currently Cache pool APIs supported only in default cluster. + @Override + public void addCachePool(CachePoolInfo info) throws IOException { + if (this.vfs == null) { + super.addCachePool(info); + return; + } + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.addCachePool(info); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public void modifyCachePool(CachePoolInfo info) throws IOException { + if (this.vfs == null) { + super.modifyCachePool(info); + return; + } + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.modifyCachePool(info); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public void removeCachePool(String poolName) throws IOException { + if (this.vfs == null) { + super.removeCachePool(poolName); + return; + } + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.removeCachePool(poolName); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public RemoteIterator listCachePools() throws IOException { + if (this.vfs == null) { + return super.listCachePools(); + } + + List childDFSs = new ArrayList<>(); + for (FileSystem fs : getChildFileSystems()) { + if (fs instanceof DistributedFileSystem) { + childDFSs.add((DistributedFileSystem) fs); + } + } + if (childDFSs.size() == 0) { + throw new UnsupportedOperationException( + "No DFS found in child fs. This API can't be supported in non DFS"); + } + return new RemoteIterator() { + int curDfsIdx = 0; + RemoteIterator currIter = + childDFSs.get(curDfsIdx++).listCachePools(); + + @Override + public boolean hasNext() throws IOException { + if (currIter.hasNext()) { + return true; + } + while (curDfsIdx < childDFSs.size()) { + currIter = childDFSs.get(curDfsIdx++).listCachePools(); + if (currIter.hasNext()) { + return true; + } + } + return false; + } + + @Override + public CachePoolEntry next() throws IOException { + if (hasNext()) { + return currIter.next(); + } + throw new java.util.NoSuchElementException("No more entries"); + } + }; + } + + @Override + public void modifyAclEntries(Path path, List aclSpec) + throws IOException { + if (this.vfs == null) { + super.modifyAclEntries(path, aclSpec); + return; + } + this.vfs.modifyAclEntries(path, aclSpec); + } + + @Override + public void removeAclEntries(Path path, List aclSpec) + throws IOException { + if (this.vfs == null) { + super.removeAclEntries(path, aclSpec); + return; + } + this.vfs.removeAclEntries(path, aclSpec); + } + + @Override + public void removeDefaultAcl(Path path) throws IOException { + if (this.vfs == null) { + super.removeDefaultAcl(path); + return; + } + this.vfs.removeDefaultAcl(path); + } + + @Override + public void removeAcl(Path path) throws IOException { + if (this.vfs == null) { + super.removeAcl(path); + return; + } + this.vfs.removeAcl(path); + } + + @Override + public void setAcl(Path path, List aclSpec) throws IOException { + if (this.vfs == null) { + super.setAcl(path, aclSpec); + return; + } + this.vfs.setAcl(path, aclSpec); + } + + @Override + public AclStatus getAclStatus(Path path) throws IOException { + if (this.vfs == null) { + return super.getAclStatus(path); + } + return this.vfs.getAclStatus(path); + } + + @Override + public void createEncryptionZone(final Path path, final String keyName) + throws IOException { + if (this.vfs == null) { + super.createEncryptionZone(path, keyName); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "createEncryptionZone"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .createEncryptionZone(mountPathInfo.getPathOnTarget(), keyName); + } + + @Override + public EncryptionZone getEZForPath(final Path path) throws IOException { + if (this.vfs == null) { + return super.getEZForPath(path); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "getEZForPath"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .getEZForPath(mountPathInfo.getPathOnTarget()); + } + + /** + * Returns the results from default DFS (fallback). If you want the results + * from specific clusters, please invoke them on child fs instance directly. + */ + @Override + public RemoteIterator listEncryptionZones() + throws IOException { + if (this.vfs == null) { + return super.listEncryptionZones(); + } + checkDefaultDFS(defaultDFS, "listEncryptionZones"); + return defaultDFS.listEncryptionZones(); + } + + @Override + public void reencryptEncryptionZone(final Path zone, + final HdfsConstants.ReencryptAction action) throws IOException { + if (this.vfs == null) { + super.reencryptEncryptionZone(zone, action); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(zone, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "reencryptEncryptionZone"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .reencryptEncryptionZone(mountPathInfo.getPathOnTarget(), action); + } + + /** + * Returns the results from default DFS (fallback). If you want the results + * from specific clusters, please invoke them on child fs instance directly. + */ + @Override + public RemoteIterator listReencryptionStatus() + throws IOException { + if (this.vfs == null) { + return super.listReencryptionStatus(); + } + checkDefaultDFS(defaultDFS, "listReencryptionStatus"); + return defaultDFS.listReencryptionStatus(); + } + + @Override + public FileEncryptionInfo getFileEncryptionInfo(final Path path) + throws IOException { + if (this.vfs == null) { + return super.getFileEncryptionInfo(path); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "getFileEncryptionInfo"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .getFileEncryptionInfo(mountPathInfo.getPathOnTarget()); + } + + @Override + public void provisionEZTrash(final Path path, + final FsPermission trashPermission) throws IOException { + if (this.vfs == null) { + super.provisionEZTrash(path, trashPermission); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "provisionEZTrash"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .provisionEZTrash(mountPathInfo.getPathOnTarget(), trashPermission); + } + + @Override + public Path provisionSnapshotTrash(final Path path, + final FsPermission trashPermission) throws IOException { + if (this.vfs == null) { + return super.provisionSnapshotTrash(path, trashPermission); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "provisionSnapshotTrash"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .provisionSnapshotTrash(mountPathInfo.getPathOnTarget(), + trashPermission); + } + + @Override + public void setXAttr(Path path, String name, byte[] value, + EnumSet flag) throws IOException { + if (this.vfs == null) { + super.setXAttr(path, name, value, flag); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + mountPathInfo.getTargetFs() + .setXAttr(mountPathInfo.getPathOnTarget(), name, value, flag); + } + + @Override + public byte[] getXAttr(Path path, String name) throws IOException { + if (this.vfs == null) { + return super.getXAttr(path, name); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + return mountPathInfo.getTargetFs() + .getXAttr(mountPathInfo.getPathOnTarget(), name); + } + + @Override + public Map getXAttrs(Path path) throws IOException { + if (this.vfs == null) { + return super.getXAttrs(path); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + return mountPathInfo.getTargetFs() + .getXAttrs(mountPathInfo.getPathOnTarget()); + } + + @Override + public Map getXAttrs(Path path, List names) + throws IOException { + if (this.vfs == null) { + return super.getXAttrs(path, names); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + return mountPathInfo.getTargetFs() + .getXAttrs(mountPathInfo.getPathOnTarget(), names); + } + + @Override + public List listXAttrs(Path path) throws IOException { + if (this.vfs == null) { + return super.listXAttrs(path); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + return mountPathInfo.getTargetFs() + .listXAttrs(mountPathInfo.getPathOnTarget()); + } + + @Override + public void removeXAttr(Path path, String name) throws IOException { + if (this.vfs == null) { + super.removeXAttr(path, name); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + mountPathInfo.getTargetFs() + .removeXAttr(mountPathInfo.getPathOnTarget(), name); + } + + @Override + public void access(Path path, FsAction mode) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + super.access(path, mode); + return; + } + this.vfs.access(path, mode); + } + + @Override + public URI getKeyProviderUri() throws IOException { + if (this.vfs == null) { + return super.getKeyProviderUri(); + } + checkDefaultDFS(defaultDFS, "getKeyProviderUri"); + return defaultDFS.getKeyProviderUri(); + } + + @Override + public KeyProvider getKeyProvider() throws IOException { + if (this.vfs == null) { + return super.getKeyProvider(); + } + checkDefaultDFS(defaultDFS, "getKeyProvider"); + return defaultDFS.getKeyProvider(); + } + + @Override + public DelegationTokenIssuer[] getAdditionalTokenIssuers() + throws IOException { + if (this.vfs == null) { + return super.getChildFileSystems(); + } + + return this.vfs.getChildFileSystems(); + } + + @Override + public DFSInotifyEventInputStream getInotifyEventStream() throws IOException { + if (this.vfs == null) { + return super.getInotifyEventStream(); + } + checkDefaultDFS(defaultDFS, "getInotifyEventStream"); + return defaultDFS.getInotifyEventStream(); + } + + @Override + public DFSInotifyEventInputStream getInotifyEventStream(long lastReadTxid) + throws IOException { + if (this.vfs == null) { + return super.getInotifyEventStream(); + } + checkDefaultDFS(defaultDFS, "getInotifyEventStream"); + return defaultDFS.getInotifyEventStream(); + } + + @Override + // DFS only API. + public void setErasureCodingPolicy(final Path path, final String ecPolicyName) + throws IOException { + if (this.vfs == null) { + super.setErasureCodingPolicy(path, ecPolicyName); + return; + } + + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "setErasureCodingPolicy"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .setErasureCodingPolicy(mountPathInfo.getPathOnTarget(), ecPolicyName); + } + + @Override + public void satisfyStoragePolicy(Path src) throws IOException { + if (this.vfs == null) { + super.satisfyStoragePolicy(src); + return; + } + this.vfs.satisfyStoragePolicy(src); + } + + @Override + public ErasureCodingPolicy getErasureCodingPolicy(final Path path) + throws IOException { + if (this.vfs == null) { + return super.getErasureCodingPolicy(path); + } + + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "getErasureCodingPolicy"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .getErasureCodingPolicy(mountPathInfo.getPathOnTarget()); + } + + /** + * Gets all erasure coding policies from all available child file systems. + */ + @Override + public Collection getAllErasureCodingPolicies() + throws IOException { + if (this.vfs == null) { + return super.getAllErasureCodingPolicies(); + } + FileSystem[] childFss = getChildFileSystems(); + List results = new ArrayList<>(); + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + for (FileSystem fs : childFss) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + results.addAll(dfs.getAllErasureCodingPolicies()); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + return results; + } + + @Override + public Map getAllErasureCodingCodecs() throws IOException { + if (this.vfs == null) { + return super.getAllErasureCodingCodecs(); + } + FileSystem[] childFss = getChildFileSystems(); + Map results = new HashMap<>(); + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + for (FileSystem fs : childFss) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + results.putAll(dfs.getAllErasureCodingCodecs()); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + return results; + } + + @Override + public AddErasureCodingPolicyResponse[] addErasureCodingPolicies( + ErasureCodingPolicy[] policies) throws IOException { + if (this.vfs == null) { + return super.addErasureCodingPolicies(policies); + } + List failedExceptions = new ArrayList<>(); + List results = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + results.addAll(Arrays.asList(dfs.addErasureCodingPolicies(policies))); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + return results.toArray(new AddErasureCodingPolicyResponse[results.size()]); + } + + @Override + public void removeErasureCodingPolicy(String ecPolicyName) + throws IOException { + if (this.vfs == null) { + super.removeErasureCodingPolicy(ecPolicyName); + return; + } + + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.removeErasureCodingPolicy(ecPolicyName); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public void enableErasureCodingPolicy(String ecPolicyName) + throws IOException { + if (this.vfs == null) { + super.enableErasureCodingPolicy(ecPolicyName); + return; + } + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.enableErasureCodingPolicy(ecPolicyName); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public void disableErasureCodingPolicy(String ecPolicyName) + throws IOException { + if (this.vfs == null) { + super.disableErasureCodingPolicy(ecPolicyName); + return; + } + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.disableErasureCodingPolicy(ecPolicyName); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public void unsetErasureCodingPolicy(final Path path) throws IOException { + if (this.vfs == null) { + super.unsetErasureCodingPolicy(path); + return; + } + + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "unsetErasureCodingPolicy"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .unsetErasureCodingPolicy(mountPathInfo.getPathOnTarget()); + } + + @Override + public ECTopologyVerifierResult getECTopologyResultForPolicies( + final String... policyNames) throws IOException { + if (this.vfs == null) { + return super.getECTopologyResultForPolicies(policyNames); + } + + List failedExceptions = new ArrayList<>(); + ECTopologyVerifierResult result = null; + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + result = dfs.getECTopologyResultForPolicies(policyNames); + if (!result.isSupported()) { + // whenever we see negative result. + return result; + } + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (result == null) { + throw new UnsupportedOperationException( + "No DFS available in child filesystems"); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + // Let's just return the last one. + return result; + } + + @Override + public Path getTrashRoot(Path path) { + if (this.vfs == null) { + return super.getTrashRoot(path); + } + return this.vfs.getTrashRoot(path); + } + + @Override + public Collection getTrashRoots(boolean allUsers) { + if (this.vfs == null) { + return super.getTrashRoots(allUsers); + } + List trashRoots = new ArrayList<>(); + for (FileSystem fs : getChildFileSystems()) { + trashRoots.addAll(fs.getTrashRoots(allUsers)); + } + return trashRoots; + } + + // Just proovided the same implementation as default in dfs as thats just + // delegated to FileSystem parent class. + @Override + protected Path fixRelativePart(Path p) { + return super.fixRelativePart(p); + } + + Statistics getFsStatistics() { + if (this.vfs == null) { + return super.getFsStatistics(); + } + return statistics; + } + + DFSOpsCountStatistics getDFSOpsCountStatistics() { + if (this.vfs == null) { + return super.getDFSOpsCountStatistics(); + } + return defaultDFS.getDFSOpsCountStatistics(); + } + + @Override + // Works only for HDFS + public HdfsDataOutputStreamBuilder createFile(Path path) { + if (this.vfs == null) { + return super.createFile(path); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = null; + try { + mountPathInfo = this.vfs.getMountPathInfo(path, getConf()); + } catch (IOException e) { + // TODO: can we return null here? + return null; + } + checkDFS(mountPathInfo.getTargetFs(), "createFile"); + return (HdfsDataOutputStreamBuilder) mountPathInfo.getTargetFs() + .createFile(mountPathInfo.getPathOnTarget()); + } + + @Deprecated + @Override + public RemoteIterator listOpenFiles() throws IOException { + if (this.vfs == null) { + return super.listOpenFiles(); + } + checkDefaultDFS(defaultDFS, "listOpenFiles"); + return defaultDFS.listOpenFiles(); + } + + @Deprecated + @Override + public RemoteIterator listOpenFiles( + EnumSet openFilesTypes) + throws IOException { + if (this.vfs == null) { + return super.listOpenFiles(openFilesTypes); + } + checkDefaultDFS(defaultDFS, "listOpenFiles"); + return defaultDFS.listOpenFiles(openFilesTypes); + } + + @Override + public RemoteIterator listOpenFiles( + EnumSet openFilesTypes, String path) + throws IOException { + if (this.vfs == null) { + return super.listOpenFiles(openFilesTypes, path); + } + Path absF = fixRelativePart(new Path(path)); + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(absF, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "listOpenFiles"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .listOpenFiles(openFilesTypes, + mountPathInfo.getPathOnTarget().toString()); + } + + @Override + public HdfsDataOutputStreamBuilder appendFile(Path path) { + if (this.vfs == null) { + return super.appendFile(path); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = null; + try { + mountPathInfo = this.vfs.getMountPathInfo(path, getConf()); + } catch (IOException e) { + LOGGER.warn("Failed to resolve the path as mount path", e); + return null; + } + checkDFS(mountPathInfo.getTargetFs(), "appendFile"); + return (HdfsDataOutputStreamBuilder) mountPathInfo.getTargetFs() + .appendFile(mountPathInfo.getPathOnTarget()); + } + + @Override + public boolean hasPathCapability(Path path, String capability) + throws IOException { + if (this.vfs == null) { + return super.hasPathCapability(path, capability); + } + return this.vfs.hasPathCapability(path, capability); + } + + //Below API provided implementations are in ViewFS but not there in DFS. + @Override + public Path resolvePath(final Path f) throws IOException { + if (this.vfs == null) { + return super.resolvePath(f); + } + return this.vfs.resolvePath(f); + } + + @Override + @SuppressWarnings("deprecation") + public boolean delete(final Path f) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + return super.delete(f); + } + return this.vfs.delete(f); + } + + @Override + public FileChecksum getFileChecksum(final Path f, final long length) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + return super.getFileChecksum(f, length); + } + return this.vfs.getFileChecksum(f, length); + } + + @Override + public boolean mkdirs(Path dir) throws IOException { + if (this.vfs == null) { + return super.mkdirs(dir); + } + return this.vfs.mkdirs(dir); + } + + @Override + public long getDefaultBlockSize(Path f) { + if (this.vfs == null) { + return super.getDefaultBlockSize(f); + } + return this.vfs.getDefaultBlockSize(f); + } + + @Override + public short getDefaultReplication(Path f) { + if (this.vfs == null) { + return super.getDefaultReplication(f); + } + return this.vfs.getDefaultReplication(f); + } + + @Override + public FsServerDefaults getServerDefaults(Path f) throws IOException { + if (this.vfs == null) { + return super.getServerDefaults(f); + } + return this.vfs.getServerDefaults(f); + } + + @Override + public void setWriteChecksum(final boolean writeChecksum) { + if (this.vfs == null) { + super.setWriteChecksum(writeChecksum); + return; + } + this.vfs.setWriteChecksum(writeChecksum); + } + + @Override + public FileSystem[] getChildFileSystems() { + if (this.vfs == null) { + return super.getChildFileSystems(); + } + return this.vfs.getChildFileSystems(); + } + + public ViewFileSystem.MountPoint[] getMountPoints() { + if (this.vfs == null) { + return null; + } + return this.vfs.getMountPoints(); + } + + @Override + public FsStatus getStatus() throws IOException { + if (this.vfs == null) { + return super.getStatus(); + } + return this.vfs.getStatus(); + } + + @Override + public long getUsed() throws IOException { + if (this.vfs == null) { + return super.getUsed(); + } + return this.vfs.getUsed(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java index 0bb995553bc7f..f6e32c40e80fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java @@ -26,9 +26,9 @@ import org.apache.hadoop.fs.XAttr.NameSpace; import org.apache.hadoop.util.StringUtils; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; @InterfaceAudience.Private public class XAttrHelper { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java index 3b3d563cd4073..716ec3af8509f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme; import org.apache.hadoop.hdfs.DFSInotifyEventInputStream; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; @@ -67,8 +68,8 @@ @InterfaceStability.Evolving public class HdfsAdmin { - private DistributedFileSystem dfs; - private static final FsPermission TRASH_PERMISSION = new FsPermission( + final private DistributedFileSystem dfs; + public static final FsPermission TRASH_PERMISSION = new FsPermission( FsAction.ALL, FsAction.ALL, FsAction.ALL, true); /** @@ -80,6 +81,10 @@ public class HdfsAdmin { */ public HdfsAdmin(URI uri, Configuration conf) throws IOException { FileSystem fs = FileSystem.get(uri, conf); + if ((fs instanceof ViewFileSystemOverloadScheme)) { + fs = ((ViewFileSystemOverloadScheme) fs) + .getRawFileSystem(new Path(FileSystem.getDefaultUri(conf)), conf); + } if (!(fs instanceof DistributedFileSystem)) { throw new IllegalArgumentException("'" + uri + "' is not an HDFS URI."); } else { @@ -165,6 +170,20 @@ public void clearQuotaByStorageType(Path src, StorageType type) throws IOExcepti */ public void allowSnapshot(Path path) throws IOException { dfs.allowSnapshot(path); + if (dfs.isSnapshotTrashRootEnabled()) { + dfs.provisionSnapshotTrash(path, TRASH_PERMISSION); + } + } + + /** + * Provision a trash directory for a given snapshottable directory. + * @param path the root of the snapshottable directory + * @return Path of the provisioned trash root + * @throws IOException if the trash directory can not be created. + */ + public Path provisionSnapshotTrash(Path path) + throws IOException { + return dfs.provisionSnapshotTrash(path, TRASH_PERMISSION); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index e8b540286c280..f858080929ebb 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -221,6 +221,12 @@ public interface HdfsClientConfigKeys { "dfs.encrypt.data.transfer.cipher.key.bitlength"; int DFS_ENCRYPT_DATA_TRANSFER_CIPHER_KEY_BITLENGTH_DEFAULT = 128; + public static final String + DFS_DATA_TRANSFER_MAX_PACKET_SIZE = + "dfs.data.transfer.max.packet.size"; + public static final int DFS_DATA_TRANSFER_MAX_PACKET_SIZE_DEFAULT = + 16 * 1024 * 1024; + String DFS_TRUSTEDCHANNEL_RESOLVER_CLASS = "dfs.trustedchannel.resolver.class"; @@ -362,6 +368,9 @@ interface Write { String EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_KEY = PREFIX + "exclude.nodes.cache.expiry.interval.millis"; long EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_DEFAULT = 10*MINUTE; + String RECOVER_LEASE_ON_CLOSE_EXCEPTION_KEY = + PREFIX + "recover.lease.on.close.exception"; + boolean RECOVER_LEASE_ON_CLOSE_EXCEPTION_DEFAULT = false; interface ByteArrayManager { String PREFIX = Write.PREFIX + "byte-array-manager."; @@ -414,6 +423,9 @@ interface Read { String PREFETCH_SIZE_KEY = PREFIX + "prefetch.size"; + String URI_CACHE_KEY = PREFIX + "uri.cache.enabled"; + boolean URI_CACHE_DEFAULT = false; + interface ShortCircuit { String PREFIX = Read.PREFIX + "shortcircuit."; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java index 02e5deb499914..5a615bbd62de4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * The Hdfs implementation of {@link FSDataInputStream}. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java index d7331299ef3e6..8af3417ca9fdf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java @@ -28,7 +28,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSOutputStream; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * The Hdfs implementation of {@link FSDataOutputStream}. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java index 028d6296e9ec6..f9fd2b1ec1670 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java @@ -30,8 +30,8 @@ import java.nio.channels.UnresolvedAddressException; import java.util.List; -import com.google.common.io.ByteArrayDataOutput; -import com.google.common.io.ByteStreams; +import org.apache.hadoop.thirdparty.com.google.common.io.ByteArrayDataOutput; +import org.apache.hadoop.thirdparty.com.google.common.io.ByteStreams; import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -74,8 +74,8 @@ import org.apache.hadoop.util.PerformanceAdvisory; import org.apache.hadoop.util.Time; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderLocal.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderLocal.java index 9c1ef461b0433..e7ddb98e700e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderLocal.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderLocal.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.client.impl; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.ReadOption; import org.apache.hadoop.fs.StorageType; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderRemote.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderRemote.java index 0f2f54c28e331..f25f50cf05411 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderRemote.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderRemote.java @@ -51,7 +51,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.DataChecksum; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java index e41b608b5b5ef..facbe70589a57 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.client.impl; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -129,6 +129,7 @@ public class DfsClientConf { private final int blockWriteLocateFollowingMaxDelayMs; private final long defaultBlockSize; private final long prefetchSize; + private final boolean uriCacheEnabled; private final short defaultReplication; private final String taskId; private final FsPermission uMask; @@ -211,24 +212,7 @@ public DfsClientConf(Configuration conf) { Write.MAX_PACKETS_IN_FLIGHT_KEY, Write.MAX_PACKETS_IN_FLIGHT_DEFAULT); - final boolean byteArrayManagerEnabled = conf.getBoolean( - Write.ByteArrayManager.ENABLED_KEY, - Write.ByteArrayManager.ENABLED_DEFAULT); - if (!byteArrayManagerEnabled) { - writeByteArrayManagerConf = null; - } else { - final int countThreshold = conf.getInt( - Write.ByteArrayManager.COUNT_THRESHOLD_KEY, - Write.ByteArrayManager.COUNT_THRESHOLD_DEFAULT); - final int countLimit = conf.getInt( - Write.ByteArrayManager.COUNT_LIMIT_KEY, - Write.ByteArrayManager.COUNT_LIMIT_DEFAULT); - final long countResetTimePeriodMs = conf.getLong( - Write.ByteArrayManager.COUNT_RESET_TIME_PERIOD_MS_KEY, - Write.ByteArrayManager.COUNT_RESET_TIME_PERIOD_MS_DEFAULT); - writeByteArrayManagerConf = new ByteArrayManager.Conf( - countThreshold, countLimit, countResetTimePeriodMs); - } + writeByteArrayManagerConf = loadWriteByteArrayManagerConf(conf); defaultBlockSize = conf.getLongBytes(DFS_BLOCK_SIZE_KEY, DFS_BLOCK_SIZE_DEFAULT); @@ -240,6 +224,10 @@ public DfsClientConf(Configuration conf) { Write.EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_DEFAULT); prefetchSize = conf.getLong(Read.PREFETCH_SIZE_KEY, 10 * defaultBlockSize); + + uriCacheEnabled = conf.getBoolean(Read.URI_CACHE_KEY, + Read.URI_CACHE_DEFAULT); + numCachedConnRetry = conf.getInt(DFS_CLIENT_CACHED_CONN_RETRY_KEY, DFS_CLIENT_CACHED_CONN_RETRY_DEFAULT); numBlockWriteRetry = conf.getInt( @@ -308,6 +296,27 @@ public DfsClientConf(Configuration conf) { "can't be more then 5."); } + private ByteArrayManager.Conf loadWriteByteArrayManagerConf( + Configuration conf) { + final boolean byteArrayManagerEnabled = conf.getBoolean( + Write.ByteArrayManager.ENABLED_KEY, + Write.ByteArrayManager.ENABLED_DEFAULT); + if (!byteArrayManagerEnabled) { + return null; + } + final int countThreshold = conf.getInt( + Write.ByteArrayManager.COUNT_THRESHOLD_KEY, + Write.ByteArrayManager.COUNT_THRESHOLD_DEFAULT); + final int countLimit = conf.getInt( + Write.ByteArrayManager.COUNT_LIMIT_KEY, + Write.ByteArrayManager.COUNT_LIMIT_DEFAULT); + final long countResetTimePeriodMs = conf.getLong( + Write.ByteArrayManager.COUNT_RESET_TIME_PERIOD_MS_KEY, + Write.ByteArrayManager.COUNT_RESET_TIME_PERIOD_MS_DEFAULT); + return new ByteArrayManager.Conf( + countThreshold, countLimit, countResetTimePeriodMs); + } + @SuppressWarnings("unchecked") private List> loadReplicaAccessorBuilderClasses(Configuration conf) { @@ -555,6 +564,13 @@ public long getPrefetchSize() { return prefetchSize; } + /** + * @return the uriCacheEnable + */ + public boolean isUriCacheEnabled() { + return uriCacheEnabled; + } + /** * @return the defaultReplication */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java index 8a040f21b8c03..d108af987cc8d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java @@ -36,7 +36,7 @@ import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/SnapshotDiffReportGenerator.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/SnapshotDiffReportGenerator.java index 4dbe98858f5d5..89e2e41352a17 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/SnapshotDiffReportGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/SnapshotDiffReportGenerator.java @@ -19,7 +19,7 @@ import java.util.*; -import com.google.common.primitives.SignedBytes; +import org.apache.hadoop.thirdparty.com.google.common.primitives.SignedBytes; import org.apache.hadoop.util.ChunkedArrayList; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/metrics/BlockReaderLocalMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/metrics/BlockReaderLocalMetrics.java index 61b497e29ca44..83bfb8b9ce5c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/metrics/BlockReaderLocalMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/metrics/BlockReaderLocalMetrics.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.client.impl.metrics; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java index edb81e9adb7ce..7bd3f969972c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java @@ -23,7 +23,7 @@ import java.util.LinkedList; import java.util.List; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.BlockStoragePolicySpi; import org.apache.hadoop.fs.StorageType; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java index e80f12aa0ee1c..3e5a4431372ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java @@ -25,7 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.Path; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.DFSUtilClient; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java index 917457f974c38..db536901283c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java @@ -26,7 +26,7 @@ import org.apache.hadoop.fs.InvalidRequestException; import org.apache.hadoop.ipc.RemoteException; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.htrace.core.TraceScope; import org.apache.htrace.core.Tracer; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 2f4dfb9b46cc1..ea90645ca082b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -727,6 +727,18 @@ BatchedDirectoryListing getBatchedListing( SnapshottableDirectoryStatus[] getSnapshottableDirListing() throws IOException; + /** + * Get listing of all the snapshots for a snapshottable directory. + * + * @return Information about all the snapshots for a snapshottable directory + * @throws IOException If an I/O error occurred + */ + @Idempotent + @ReadOnly(isCoordinated = true) + SnapshotStatus[] getSnapshotListing(String snapshotRoot) + throws IOException; + + /////////////////////////////////////// // System issues and management /////////////////////////////////////// diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java index 0278fc92d2e2f..2cb16879c3f6f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.net.InetSocketAddress; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java index 3559ab97d8fe1..02c4f9a1f21b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.protocol; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java index c8a2722621020..48b581dfe6acb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.protocol; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index a025b9bad2e69..23a5d33a72f48 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.util.StringUtils; @InterfaceAudience.Private @@ -37,6 +38,8 @@ public final class HdfsConstants { public static final byte MEMORY_STORAGE_POLICY_ID = 15; public static final String MEMORY_STORAGE_POLICY_NAME = "LAZY_PERSIST"; + public static final byte ALLNVDIMM_STORAGE_POLICY_ID = 14; + public static final String ALLNVDIMM_STORAGE_POLICY_NAME = "ALL_NVDIMM"; public static final byte ALLSSD_STORAGE_POLICY_ID = 12; public static final String ALLSSD_STORAGE_POLICY_NAME = "ALL_SSD"; public static final byte ONESSD_STORAGE_POLICY_ID = 10; @@ -64,6 +67,7 @@ public enum StoragePolicy{ HOT(HOT_STORAGE_POLICY_ID), ONE_SSD(ONESSD_STORAGE_POLICY_ID), ALL_SSD(ALLSSD_STORAGE_POLICY_ID), + ALL_NVDIMM(ALLNVDIMM_STORAGE_POLICY_ID), LAZY_PERSIST(MEMORY_STORAGE_POLICY_ID); private byte value; @@ -85,6 +89,8 @@ public static StoragePolicy valueOf(int value) { return ONE_SSD; case 12: return ALL_SSD; + case 14: + return ALL_NVDIMM; case 15: return LAZY_PERSIST; default: @@ -158,7 +164,7 @@ public byte value() { * period, no other client can write to the file. The writing client can * periodically renew the lease. When the file is closed, the lease is * revoked. The lease duration is bound by this soft limit and a - * {@link HdfsConstants#LEASE_HARDLIMIT_PERIOD hard limit}. Until the + * {@link HdfsClientConfigKeys#DFS_LEASE_HARDLIMIT_KEY }. Until the * soft limit expires, the writer has sole write access to the file. If the * soft limit expires and the client fails to close the file or renew the * lease, another client can preempt the lease. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPartialListing.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPartialListing.java index c3e0ac585ebfd..d96c7892b76ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPartialListing.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPartialListing.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.protocol; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.ipc.RemoteException; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java index ed4d668806c16..f2d8135ab4dee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java @@ -22,14 +22,14 @@ import java.util.Comparator; import java.util.List; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.security.token.Token; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * Associates a block with the Datanodes that contain its replicas diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java index dabeceacc3e3c..5d7b91343f239 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hdfs.protocol; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReencryptionInfoProto; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java index 7bc95c9350521..e6f20c9ce1b82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java @@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path; -import com.google.common.base.Objects; +import org.apache.hadoop.thirdparty.com.google.common.base.Objects; import org.apache.hadoop.hdfs.DFSUtilClient; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java index 79fd543c3be90..74329bc1e8a12 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java @@ -20,7 +20,7 @@ import java.util.Collections; import java.util.List; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.DFSUtilClient; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java new file mode 100644 index 0000000000000..8f10db8b8e62d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java @@ -0,0 +1,205 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import java.io.PrintStream; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.EnumSet; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSUtilClient; + +/** + * Metadata about a snapshottable directory. + */ +public class SnapshotStatus { + /** + * Basic information of the snapshot directory. + */ + private final HdfsFileStatus dirStatus; + + /** + * Snapshot ID for the snapshot. + */ + private final int snapshotID; + + /** + * Whether the snapshot is deleted or not. + */ + private final boolean isDeleted; + + /** + * Full path of the parent. + */ + private byte[] parentFullPath; + + public SnapshotStatus(long modificationTime, long accessTime, + FsPermission permission, + EnumSet flags, + String owner, String group, byte[] localName, + long inodeId, int childrenNum, int snapshotID, + boolean isDeleted, byte[] parentFullPath) { + this.dirStatus = new HdfsFileStatus.Builder() + .isdir(true) + .mtime(modificationTime) + .atime(accessTime) + .perm(permission) + .flags(flags) + .owner(owner) + .group(group) + .path(localName) + .fileId(inodeId) + .children(childrenNum) + .build(); + this.snapshotID = snapshotID; + this.isDeleted = isDeleted; + this.parentFullPath = parentFullPath; + } + + public SnapshotStatus(HdfsFileStatus dirStatus, + int snapshotID, boolean isDeleted, + byte[] parentFullPath) { + this.dirStatus = dirStatus; + this.snapshotID = snapshotID; + this.isDeleted = isDeleted; + this.parentFullPath = parentFullPath; + } + + /** + * sets the path name. + * @param path path + */ + public void setParentFullPath(byte[] path) { + parentFullPath = path; + } + + /** + * @return snapshot id for the snapshot + */ + public int getSnapshotID() { + return snapshotID; + } + + /** + * @return whether snapshot is deleted + */ + public boolean isDeleted() { + return isDeleted; + } + + /** + * @return The basic information of the directory + */ + public HdfsFileStatus getDirStatus() { + return dirStatus; + } + + /** + * @return Full path of the file + */ + public byte[] getParentFullPath() { + return parentFullPath; + } + + /** + * @return Full path of the snapshot + */ + public Path getFullPath() { + String parentFullPathStr = + (parentFullPath == null || parentFullPath.length == 0) ? + "/" : DFSUtilClient.bytes2String(parentFullPath); + return new Path(getSnapshotPath(parentFullPathStr, + dirStatus.getLocalName())); + } + + /** + * Print a list of {@link SnapshotStatus} out to a given stream. + * + * @param stats The list of {@link SnapshotStatus} + * @param out The given stream for printing. + */ + public static void print(SnapshotStatus[] stats, + PrintStream out) { + if (stats == null || stats.length == 0) { + out.println(); + return; + } + int maxRepl = 0, maxLen = 0, maxOwner = 0, maxGroup = 0; + int maxSnapshotID = 0; + for (SnapshotStatus status : stats) { + maxRepl = maxLength(maxRepl, status.dirStatus.getReplication()); + maxLen = maxLength(maxLen, status.dirStatus.getLen()); + maxOwner = maxLength(maxOwner, status.dirStatus.getOwner()); + maxGroup = maxLength(maxGroup, status.dirStatus.getGroup()); + maxSnapshotID = maxLength(maxSnapshotID, status.snapshotID); + } + + String lineFormat = "%s%s " // permission string + + "%" + maxRepl + "s " + + (maxOwner > 0 ? "%-" + maxOwner + "s " : "%s") + + (maxGroup > 0 ? "%-" + maxGroup + "s " : "%s") + + "%" + maxLen + "s " + + "%s " // mod time + + "%" + maxSnapshotID + "s " + + "%s " // deletion status + + "%s"; // path + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm"); + + for (SnapshotStatus status : stats) { + String line = String.format(lineFormat, "d", + status.dirStatus.getPermission(), + status.dirStatus.getReplication(), + status.dirStatus.getOwner(), + status.dirStatus.getGroup(), + String.valueOf(status.dirStatus.getLen()), + dateFormat.format(new Date(status.dirStatus.getModificationTime())), + status.snapshotID, + status.isDeleted ? "DELETED" : "ACTIVE", + getSnapshotPath(DFSUtilClient.bytes2String(status.parentFullPath), + status.dirStatus.getLocalName()) + ); + out.println(line); + } + } + + private static int maxLength(int n, Object value) { + return Math.max(n, String.valueOf(value).length()); + } + + public static String getSnapshotPath(String snapshottableDir, + String snapshotRelativePath) { + String parentFullPathStr = + snapshottableDir == null || snapshottableDir.isEmpty() ? + "/" : snapshottableDir; + final StringBuilder b = new StringBuilder(parentFullPathStr); + if (b.charAt(b.length() - 1) != Path.SEPARATOR_CHAR) { + b.append(Path.SEPARATOR); + } + return b.append(HdfsConstants.DOT_SNAPSHOT_DIR) + .append(Path.SEPARATOR) + .append(snapshotRelativePath) + .toString(); + } + + public static String getParentPath(String snapshotPath) { + int index = snapshotPath.indexOf(HdfsConstants.DOT_SNAPSHOT_DIR); + return index == -1 ? snapshotPath : snapshotPath.substring(0, index - 1); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java index c7628d8dd91df..10884f27f90a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.protocol; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReencryptionInfoProto; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketHeader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketHeader.java index 486909466474c..cc958e35df116 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketHeader.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketHeader.java @@ -27,9 +27,9 @@ import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.PacketHeaderProto; import org.apache.hadoop.hdfs.util.ByteBufferOutputStream; -import com.google.common.base.Preconditions; -import com.google.common.primitives.Shorts; -import com.google.common.primitives.Ints; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Shorts; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java index 6b717ecdfbff2..6949a9d40a656 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java @@ -25,11 +25,14 @@ import java.nio.channels.ReadableByteChannel; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.util.DirectBufferPool; import org.apache.hadoop.io.IOUtils; -import com.google.common.base.Preconditions; -import com.google.common.primitives.Ints; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +48,7 @@ public class PacketReceiver implements Closeable { * The max size of any single packet. This prevents OOMEs when * invalid data is sent. */ - public static final int MAX_PACKET_SIZE = 16 * 1024 * 1024; + public static final int MAX_PACKET_SIZE; static final Logger LOG = LoggerFactory.getLogger(PacketReceiver.class); @@ -74,6 +77,13 @@ public class PacketReceiver implements Closeable { */ private PacketHeader curHeader; + static { + Configuration conf = new HdfsConfiguration(); + MAX_PACKET_SIZE = conf.getInt(HdfsClientConfigKeys. + DFS_DATA_TRANSFER_MAX_PACKET_SIZE, + HdfsClientConfigKeys.DFS_DATA_TRANSFER_MAX_PACKET_SIZE_DEFAULT); + } + public PacketReceiver(boolean useDirectBuffers) { this.useDirectBuffers = useDirectBuffers; reallocPacketBuf(PacketHeader.PKT_LENGTHS_LEN); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java index 6992477496754..b58fbb8992a47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java @@ -24,7 +24,7 @@ import java.io.OutputStream; import java.util.ArrayList; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java index 8d6e318168b3d..526f3d0c66512 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java @@ -57,10 +57,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Charsets; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import com.google.common.net.InetAddresses; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses; import org.apache.hadoop.thirdparty.protobuf.ByteString; /** @@ -305,20 +305,22 @@ public static SaslMessageWithHandshake readSaslMessageWithHandshakeSecret( public static CipherOption negotiateCipherOption(Configuration conf, List options) throws IOException { // Negotiate cipher suites if configured. Currently, the only supported - // cipher suite is AES/CTR/NoPadding, but the protocol allows multiple - // values for future expansion. + // cipher suite is AES/CTR/NoPadding or SM4/CTR/NoPadding, but the protocol + // allows multiple values for future expansion. String cipherSuites = conf.get(DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY); if (cipherSuites == null || cipherSuites.isEmpty()) { return null; } - if (!cipherSuites.equals(CipherSuite.AES_CTR_NOPADDING.getName())) { + if (!cipherSuites.equals(CipherSuite.AES_CTR_NOPADDING.getName()) && + !cipherSuites.equals(CipherSuite.SM4_CTR_NOPADDING.getName())) { throw new IOException(String.format("Invalid cipher suite, %s=%s", DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY, cipherSuites)); } if (options != null) { for (CipherOption option : options) { CipherSuite suite = option.getCipherSuite(); - if (suite == CipherSuite.AES_CTR_NOPADDING) { + if (suite == CipherSuite.AES_CTR_NOPADDING || + suite == CipherSuite.SM4_CTR_NOPADDING) { int keyLen = conf.getInt( DFS_ENCRYPT_DATA_TRANSFER_CIPHER_KEY_BITLENGTH_KEY, DFS_ENCRYPT_DATA_TRANSFER_CIPHER_KEY_BITLENGTH_DEFAULT) / 8; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java index acd1e505cbbc0..cf6a76a5187bc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java @@ -21,7 +21,7 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_ENCRYPT_DATA_OVERWRITE_DOWNSTREAM_NEW_QOP_KEY; import static org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil.*; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -64,8 +64,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Charsets; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * Negotiates SASL for DataTransferProtocol on behalf of a client. There are @@ -548,14 +548,19 @@ private IOStreamPair doSaslHandshake(InetAddress addr, DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY); if (requestedQopContainsPrivacy(saslProps)) { // Negotiate cipher suites if configured. Currently, the only supported - // cipher suite is AES/CTR/NoPadding, but the protocol allows multiple - // values for future expansion. + // cipher suite is AES/CTR/NoPadding or SM4/CTR/Nopadding, + // but the protocol allows multiple values for future expansion. if (cipherSuites != null && !cipherSuites.isEmpty()) { - if (!cipherSuites.equals(CipherSuite.AES_CTR_NOPADDING.getName())) { + CipherOption option = null; + if (cipherSuites.equals(CipherSuite.AES_CTR_NOPADDING.getName())) { + option = new CipherOption(CipherSuite.AES_CTR_NOPADDING); + } else if (cipherSuites.equals( + CipherSuite.SM4_CTR_NOPADDING.getName())) { + option = new CipherOption(CipherSuite.SM4_CTR_NOPADDING); + } else { throw new IOException(String.format("Invalid cipher suite, %s=%s", DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY, cipherSuites)); } - CipherOption option = new CipherOption(CipherSuite.AES_CTR_NOPADDING); cipherOptions = Lists.newArrayListWithCapacity(1); cipherOptions.add(option); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 7e41460ca4c63..0886636419b67 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -24,7 +24,7 @@ import java.util.HashMap; import java.util.List; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -89,6 +89,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusRequestProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusResponseProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.ModifyAclEntriesRequestProto; @@ -150,6 +151,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportListingResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshottableDirListingRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshottableDirListingResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotListingRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotListingResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePoliciesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePoliciesResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePolicyRequestProto; @@ -1299,6 +1302,25 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() } } + @Override + public SnapshotStatus[] getSnapshotListing(String path) + throws IOException { + GetSnapshotListingRequestProto req = + GetSnapshotListingRequestProto.newBuilder() + .setSnapshotRoot(path).build(); + try { + GetSnapshotListingResponseProto result = rpcProxy + .getSnapshotListing(null, req); + + if (result.hasSnapshotList()) { + return PBHelperClient.convert(result.getSnapshotList()); + } + return null; + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + @Override public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot, String fromSnapshot, String toSnapshot) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 9fc302464271d..bacd45d7b38d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -28,12 +28,12 @@ import java.util.Map; import java.util.Set; -import com.google.common.base.Preconditions; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.collect.Lists; -import com.google.common.primitives.Shorts; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; +import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Shorts; import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; @@ -114,6 +114,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryScopeProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryTypeProto; @@ -184,6 +185,8 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryListingProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryStatusProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotListingProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotStatusProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageReportProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypesProto; @@ -472,6 +475,8 @@ public static StorageTypeProto convertStorageType(StorageType type) { return StorageTypeProto.RAM_DISK; case PROVIDED: return StorageTypeProto.PROVIDED; + case NVDIMM: + return StorageTypeProto.NVDIMM; default: throw new IllegalStateException( "BUG: StorageType not found, type=" + type); @@ -490,6 +495,8 @@ public static StorageType convertStorageType(StorageTypeProto type) { return StorageType.RAM_DISK; case PROVIDED: return StorageType.PROVIDED; + case NVDIMM: + return StorageType.NVDIMM; default: throw new IllegalStateException( "BUG: StorageTypeProto not found, type=" + type); @@ -558,6 +565,8 @@ public static CipherSuite convert(HdfsProtos.CipherSuiteProto proto) { switch (proto) { case AES_CTR_NOPADDING: return CipherSuite.AES_CTR_NOPADDING; + case SM4_CTR_NOPADDING: + return CipherSuite.SM4_CTR_NOPADDING; default: // Set to UNKNOWN and stash the unknown enum value CipherSuite suite = CipherSuite.UNKNOWN; @@ -596,6 +605,8 @@ public static HdfsProtos.CipherSuiteProto convert(CipherSuite suite) { return HdfsProtos.CipherSuiteProto.UNKNOWN; case AES_CTR_NOPADDING: return HdfsProtos.CipherSuiteProto.AES_CTR_NOPADDING; + case SM4_CTR_NOPADDING: + return HdfsProtos.CipherSuiteProto.SM4_CTR_NOPADDING; default: return null; } @@ -1669,6 +1680,49 @@ public static SnapshottableDirectoryStatus convert( sdirStatusProto.getParentFullpath().toByteArray()); } + public static SnapshotStatus[] convert( + HdfsProtos.SnapshotListingProto sdlp) { + if (sdlp == null) { + return null; + } + List list = sdlp + .getSnapshotListingList(); + if (list.isEmpty()) { + return new SnapshotStatus[0]; + } else { + SnapshotStatus[] result = + new SnapshotStatus[list.size()]; + for (int i = 0; i < list.size(); i++) { + result[i] = convert(list.get(i)); + } + return result; + } + } + + public static SnapshotStatus convert( + HdfsProtos.SnapshotStatusProto sdirStatusProto) { + if (sdirStatusProto == null) { + return null; + } + final HdfsFileStatusProto status = sdirStatusProto.getDirStatus(); + EnumSet flags = status.hasFlags() + ? convertFlags(status.getFlags()) + : convertFlags(status.getPermission()); + return new SnapshotStatus( + status.getModificationTime(), + status.getAccessTime(), + convert(status.getPermission()), + flags, + status.getOwner(), + status.getGroup(), + status.getPath().toByteArray(), + status.getFileId(), + status.getChildrenNum(), + sdirStatusProto.getSnapshotID(), + sdirStatusProto.getIsDeleted(), + sdirStatusProto.getParentFullpath().toByteArray()); + } + // DataEncryptionKey public static DataEncryptionKey convert(DataEncryptionKeyProto bet) { String encryptionAlgorithm = bet.getEncryptionAlgorithm(); @@ -1907,7 +1961,7 @@ public static StorageReport convert(StorageReportProto p) { return new StorageReport(p.hasStorage() ? convert(p.getStorage()) : new DatanodeStorage(p.getStorageUuid()), p.getFailed(), p.getCapacity(), p.getDfsUsed(), p.getRemaining(), - p.getBlockPoolUsed(), nonDfsUsed); + p.getBlockPoolUsed(), nonDfsUsed, p.getMount()); } public static DatanodeStorage convert(DatanodeStorageProto s) { @@ -2078,7 +2132,8 @@ public static FsServerDefaults convert(FsServerDefaultsProto fs) { fs.getTrashInterval(), convert(fs.getChecksumType()), fs.hasKeyProviderUri() ? fs.getKeyProviderUri() : null, - (byte) fs.getPolicyId()); + (byte) fs.getPolicyId(), + fs.getSnapshotTrashRootEnabled()); } public static List convert( @@ -2252,7 +2307,8 @@ public static FsServerDefaultsProto convert(FsServerDefaults fs) { .setEncryptDataTransfer(fs.getEncryptDataTransfer()) .setTrashInterval(fs.getTrashInterval()) .setChecksumType(convert(fs.getChecksumType())) - .setPolicyId(fs.getDefaultStoragePolicyId()); + .setPolicyId(fs.getDefaultStoragePolicyId()) + .setSnapshotTrashRootEnabled(fs.getSnapshotTrashRootEnabled()); if (fs.getKeyProviderUri() != null) { builder.setKeyProviderUri(fs.getKeyProviderUri()); } @@ -2367,6 +2423,24 @@ public static SnapshottableDirectoryStatusProto convert( return builder.build(); } + public static HdfsProtos.SnapshotStatusProto convert(SnapshotStatus status) { + if (status == null) { + return null; + } + byte[] parentFullPath = status.getParentFullPath(); + ByteString parentFullPathBytes = getByteString( + parentFullPath == null ? DFSUtilClient.EMPTY_BYTES : parentFullPath); + HdfsFileStatusProto fs = convert(status.getDirStatus()); + HdfsProtos.SnapshotStatusProto.Builder builder = + HdfsProtos.SnapshotStatusProto + .newBuilder() + .setSnapshotID(status.getSnapshotID()) + .setParentFullpath(parentFullPathBytes) + .setIsDeleted(status.isDeleted()) + .setDirStatus(fs); + return builder.build(); + } + public static HdfsFileStatusProto[] convert(HdfsFileStatus[] fs) { if (fs == null) return null; final int len = fs.length; @@ -2622,7 +2696,8 @@ public static StorageReportProto convert(StorageReport r) { .setDfsUsed(r.getDfsUsed()).setRemaining(r.getRemaining()) .setStorageUuid(r.getStorage().getStorageID()) .setStorage(convert(r.getStorage())) - .setNonDfsUsed(r.getNonDfsUsed()); + .setNonDfsUsed(r.getNonDfsUsed()) + .setMount(r.getMount()); return builder.build(); } @@ -2649,6 +2724,21 @@ public static SnapshottableDirectoryListingProto convert( .addAllSnapshottableDirListing(protoList).build(); } + public static HdfsProtos.SnapshotListingProto convert( + SnapshotStatus[] status) { + if (status == null) { + return null; + } + HdfsProtos.SnapshotStatusProto[] protos = + new HdfsProtos.SnapshotStatusProto[status.length]; + for (int i = 0; i < status.length; i++) { + protos[i] = convert(status[i]); + } + List protoList = Arrays.asList(protos); + return SnapshotListingProto.newBuilder() + .addAllSnapshotListing(protoList).build(); + } + public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) { if (entry == null) { return null; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ReconfigurationProtocolUtils.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ReconfigurationProtocolUtils.java index a48c401927406..e387ca7d843d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ReconfigurationProtocolUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ReconfigurationProtocolUtils.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ReconfigurationProtocolProtos.GetReconfigurationStatusConfigChangeProto; import org.apache.hadoop.hdfs.protocol.proto.ReconfigurationProtocolProtos.GetReconfigurationStatusResponseProto; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; /** * This is a client side utility class that handles diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java index 1d395ec24ac72..4b2e6cd989707 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java @@ -27,7 +27,7 @@ import java.util.EnumSet; import java.util.Optional; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.AccessModeProto; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java index d753174e71bcc..dcbf6c8f0b92c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java @@ -33,7 +33,7 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * A delegation token identifier that is specific to HDFS. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockMetadataHeader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockMetadataHeader.java index 2d1cfc135521f..c343d00e61371 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockMetadataHeader.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockMetadataHeader.java @@ -33,7 +33,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.util.DataChecksum; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.util.InvalidChecksumSizeException; import org.slf4j.Logger; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java index 505273ec9790f..130e8c1c9c728 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java index af34c49dd12bb..7e3954c562a58 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.SerializationFeature; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java index 6b99a7f7ce23b..9cabeb9037fc5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java @@ -50,7 +50,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * A {@link org.apache.hadoop.io.retry.FailoverProxyProvider} implementation diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java index 496389a1ddc87..95078a7412d74 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.protocol; -import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowPeerReports.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowPeerReports.java index 218e30df4d2dc..a3b3445d5a743 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowPeerReports.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowPeerReports.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.protocol; -import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java index 7042b58a2f4b2..7c189b1f39fea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java @@ -28,11 +28,19 @@ public class StorageReport { private final long nonDfsUsed; private final long remaining; private final long blockPoolUsed; + private final String mount; public static final StorageReport[] EMPTY_ARRAY = {}; public StorageReport(DatanodeStorage storage, boolean failed, long capacity, long dfsUsed, long remaining, long bpUsed, long nonDfsUsed) { + this(storage, failed, capacity, dfsUsed, + remaining, bpUsed, nonDfsUsed, ""); + } + + public StorageReport(DatanodeStorage storage, boolean failed, long capacity, + long dfsUsed, long remaining, long bpUsed, + long nonDfsUsed, String mount) { this.storage = storage; this.failed = failed; this.capacity = capacity; @@ -40,6 +48,7 @@ public StorageReport(DatanodeStorage storage, boolean failed, long capacity, this.nonDfsUsed = nonDfsUsed; this.remaining = remaining; this.blockPoolUsed = bpUsed; + this.mount = mount; } public DatanodeStorage getStorage() { @@ -69,4 +78,8 @@ public long getRemaining() { public long getBlockPoolUsed() { return blockPoolUsed; } + + public String getMount() { + return mount; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java index 81cc68da07295..7bf768935d9d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java @@ -26,7 +26,7 @@ import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.net.unix.DomainSocketWatcher; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * DfsClientShm is a subclass of ShortCircuitShm which is used by the diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java index dfce2304039af..7cb0c3853ce65 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java @@ -43,8 +43,8 @@ import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.net.unix.DomainSocketWatcher; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DomainSocketFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DomainSocketFactory.java index 760e920c232b5..40436395887a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DomainSocketFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DomainSocketFactory.java @@ -21,7 +21,7 @@ import java.net.InetSocketAddress; import java.util.concurrent.TimeUnit; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.IOUtils; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.hdfs.DFSUtilClient; @@ -30,9 +30,9 @@ import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.util.PerformanceAdvisory; -import com.google.common.base.Preconditions; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.cache.Cache; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java index a9970c2f3adaa..222cfe245bc46 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java @@ -54,9 +54,9 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Waitable; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitReplica.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitReplica.java index 14116e2fdc1c6..86218aa0fcd8d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitReplica.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitReplica.java @@ -31,8 +31,8 @@ import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.util.Time; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java index b9fcadae529b3..1cb123bb58f3b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java @@ -39,9 +39,9 @@ import sun.misc.Unsafe; -import com.google.common.base.Preconditions; -import com.google.common.collect.ComparisonChain; -import com.google.common.primitives.Ints; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; import javax.annotation.Nonnull; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java index 164b03bf203a3..059280e494678 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java @@ -26,7 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.util.Time; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java index 012d70895021d..460c611ef88eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.util; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.StorageType; @@ -28,7 +28,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.DFSStripedOutputStream; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java index 31de804d74ee9..c83ccf9d738ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java @@ -31,8 +31,8 @@ import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.FSInputStream; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.net.HttpHeaders; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.net.HttpHeaders; import javax.annotation.Nonnull; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java index e846b56200993..fd31da3ae5bf0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java @@ -19,9 +19,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; @@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -872,4 +873,39 @@ private static SnapshottableDirectoryStatus toSnapshottableDirectoryStatus( snapshotQuota, parentFullPath); return snapshottableDirectoryStatus; } + + public static SnapshotStatus[] toSnapshotList(final Map json) { + if (json == null) { + return null; + } + List list = (List) json.get("SnapshotList"); + if (list == null) { + return null; + } + SnapshotStatus[] statuses = + new SnapshotStatus[list.size()]; + for (int i = 0; i < list.size(); i++) { + statuses[i] = toSnapshotStatus((Map) list.get(i)); + } + return statuses; + } + + private static SnapshotStatus toSnapshotStatus( + Map json) { + if (json == null) { + return null; + } + int snapshotID = getInt(json, "snapshotID", 0); + boolean isDeleted = "DELETED".equalsIgnoreCase( + (String)json.get("deletionStatus")); + String fullPath = ((String) json.get("fullPath")); + + HdfsFileStatus dirStatus = + toFileStatus((Map) json.get("dirStatus"), false); + SnapshotStatus snapshotStatus = + new SnapshotStatus(dirStatus, snapshotID, + isDeleted, DFSUtilClient.string2Bytes( + SnapshotStatus.getParentPath(fullPath))); + return snapshotStatus; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java index 870103eddf021..dfc754b3a819e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java @@ -38,7 +38,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * This class implements the aspects that relate to delegation tokens for all diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java index 8b6c7f7cfd02a..589afb4604d3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java @@ -34,7 +34,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Utilities for handling URLs diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 25e7f7373226b..080fba87e83d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -106,6 +106,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FileEncryptionInfoProto; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -134,10 +135,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Charsets; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; @@ -1459,6 +1460,19 @@ SnapshottableDirectoryStatus[] decodeResponse(Map json) { }.run(); } + public SnapshotStatus[] getSnapshotListing(final Path snapshotDir) + throws IOException { + storageStatistics + .incrementOpCounter(OpType.GET_SNAPSHOT_LIST); + final HttpOpParam.Op op = GetOpParam.Op.GETSNAPSHOTLIST; + return new FsPathResponseRunner(op, snapshotDir) { + @Override + SnapshotStatus[] decodeResponse(Map json) { + return JsonUtilClient.toSnapshotList(json); + } + }.run(); + } + @Override public boolean setReplication(final Path p, final short replication ) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java index 9ab3ad57fa8e2..5419219b6effc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java @@ -24,7 +24,7 @@ import java.util.List; import java.util.regex.Pattern; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.permission.AclEntry; /** AclPermission parameter. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java index f9a5fa6de4530..abbef9d8e9b91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java @@ -62,7 +62,8 @@ public enum Op implements HttpOpParam.Op { LISTSTATUS_BATCH(false, HttpURLConnection.HTTP_OK), GETSERVERDEFAULTS(false, HttpURLConnection.HTTP_OK), GETSNAPSHOTDIFF(false, HttpURLConnection.HTTP_OK), - GETSNAPSHOTTABLEDIRECTORYLIST(false, HttpURLConnection.HTTP_OK); + GETSNAPSHOTTABLEDIRECTORYLIST(false, HttpURLConnection.HTTP_OK), + GETSNAPSHOTLIST(false, HttpURLConnection.HTTP_OK); final boolean redirect; final int expectedHttpResponseCode; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java index 19efb9d1818c5..0ebafe747fc28 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java @@ -19,7 +19,7 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT; import org.apache.hadoop.security.UserGroupInformation; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.text.MessageFormat; import java.util.regex.Pattern; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto index 3fb57bc02d0ac..20967cc13ab86 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto @@ -303,6 +303,14 @@ message GetSnapshottableDirListingResponseProto { optional SnapshottableDirectoryListingProto snapshottableDirList = 1; } +message GetSnapshotListingRequestProto { + required string snapshotRoot = 1; +} + +message GetSnapshotListingResponseProto { + optional SnapshotListingProto snapshotList = 1; +} + message GetSnapshotDiffReportRequestProto { required string snapshotRoot = 1; required string fromSnapshot = 2; @@ -986,6 +994,8 @@ service ClientNamenodeProtocol { returns(DisallowSnapshotResponseProto); rpc getSnapshottableDirListing(GetSnapshottableDirListingRequestProto) returns(GetSnapshottableDirListingResponseProto); + rpc getSnapshotListing(GetSnapshotListingRequestProto) + returns(GetSnapshotListingResponseProto); rpc deleteSnapshot(DeleteSnapshotRequestProto) returns(DeleteSnapshotResponseProto); rpc getSnapshotDiffReport(GetSnapshotDiffReportRequestProto) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index 82fe329c9ce5e..5ae58cfddad5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -158,6 +158,7 @@ message StorageReportProto { optional uint64 blockPoolUsed = 6 [ default = 0 ]; optional DatanodeStorageProto storage = 7; // supersedes StorageUuid optional uint64 nonDfsUsed = 8; + optional string mount = 9; } /** @@ -221,6 +222,7 @@ enum StorageTypeProto { ARCHIVE = 3; RAM_DISK = 4; PROVIDED = 5; + NVDIMM = 6; } /** @@ -295,6 +297,7 @@ message DataEncryptionKeyProto { enum CipherSuiteProto { UNKNOWN = 1; AES_CTR_NOPADDING = 2; + SM4_CTR_NOPADDING = 3; } /** @@ -526,6 +529,7 @@ message FsServerDefaultsProto { optional ChecksumTypeProto checksumType = 8 [default = CHECKSUM_CRC32]; optional string keyProviderUri = 9; optional uint32 policyId = 10 [default = 0]; + optional bool snapshotTrashRootEnabled = 11 [default = false]; } @@ -563,6 +567,21 @@ message SnapshottableDirectoryStatusProto { required bytes parent_fullpath = 4; } +/** + * Status of a snapshot directory: besides the normal information for + * a directory status, also include snapshot ID, and + * the full path of the parent directory. + */ +message SnapshotStatusProto { + required HdfsFileStatusProto dirStatus = 1; + + // Fields specific for snapshot directory + required uint32 snapshotID = 2; + required bytes parent_fullpath = 3; + required bool isDeleted = 4; +} + + /** * Snapshottable directory listing */ @@ -570,6 +589,13 @@ message SnapshottableDirectoryListingProto { repeated SnapshottableDirectoryStatusProto snapshottableDirListing = 1; } +/** + * Snapshot listing + */ +message SnapshotListingProto { + repeated SnapshotStatusProto snapshotListing = 1; +} + /** * Snapshot diff report entry */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java index b24df2bfce929..3f1ff8826957e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs; -import com.google.common.collect.HashMultiset; +import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultiset; import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.net.unix.DomainSocket; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java index 41069b439784f..4e6f4e3f4ba38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java @@ -41,6 +41,7 @@ public class TestReadOnly { "getListing", "getBatchedListing", "getSnapshottableDirListing", + "getSnapshotListing", "getPreferredBlockSize", "listCorruptFileBlocks", "getFileInfo", diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java index 5704ef0bd4c03..23e7b74469c8d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java @@ -56,7 +56,7 @@ import static org.mockito.Mockito.when; import static org.mockito.Mockito.mock; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; public class TestRequestHedgingProxyProvider { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/util/TestECPolicyLoader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/util/TestECPolicyLoader.java index a6adb97dac61c..55fa56784f524 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/util/TestECPolicyLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/util/TestECPolicyLoader.java @@ -52,12 +52,12 @@ public void testLoadECPolicy() throws Exception { out.println("1"); out.println(""); out.println(" "); - out.println(" RS"); + out.println(" rs"); out.println(" 12"); out.println(" 4"); out.println(" "); out.println(" "); - out.println(" RS-legacy"); + out.println(" rs-legacy"); out.println(" 12"); out.println(" 4"); out.println(" "); @@ -87,7 +87,7 @@ public void testLoadECPolicy() throws Exception { assertEquals(0, schema1.getExtraOptions().size()); assertEquals(12, schema1.getNumDataUnits()); assertEquals(4, schema1.getNumParityUnits()); - assertEquals("RS", schema1.getCodecName()); + assertEquals("rs", schema1.getCodecName()); ErasureCodingPolicy policy2 = policies.get(1); ECSchema schema2 = policy2.getSchema(); @@ -95,7 +95,7 @@ public void testLoadECPolicy() throws Exception { assertEquals(0, schema2.getExtraOptions().size()); assertEquals(12, schema2.getNumDataUnits()); assertEquals(4, schema2.getNumParityUnits()); - assertEquals("RS-legacy", schema2.getCodecName()); + assertEquals("rs-legacy", schema2.getCodecName()); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestByteRangeInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestByteRangeInputStream.java index 6c426f575b91c..010d7c5870c54 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestByteRangeInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestByteRangeInputStream.java @@ -34,7 +34,7 @@ import java.net.HttpURLConnection; import java.net.URL; -import com.google.common.net.HttpHeaders; +import org.apache.hadoop.thirdparty.com.google.common.net.HttpHeaders; import org.apache.hadoop.hdfs.web.ByteRangeInputStream.InputStreamAndFileLength; import org.apache.hadoop.test.Whitebox; import org.junit.Test; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestURLConnectionFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestURLConnectionFactory.java index 2be8bf4336279..108ce50420640 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestURLConnectionFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestURLConnectionFactory.java @@ -33,7 +33,7 @@ import org.junit.Assert; import org.junit.Test; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.slf4j.LoggerFactory; public final class TestURLConnectionFactory { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSOAuth2.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSOAuth2.java index bd12204118694..af17d0b1e5ee3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSOAuth2.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSOAuth2.java @@ -31,7 +31,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.mockserver.client.server.MockServerClient; +import org.mockserver.client.MockServerClient; import org.mockserver.integration.ClientAndServer; import org.mockserver.model.Header; import org.mockserver.model.HttpRequest; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestClientCredentialTimeBasedTokenRefresher.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestClientCredentialTimeBasedTokenRefresher.java index 935cd9eeb665b..9ae7ff88fb6b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestClientCredentialTimeBasedTokenRefresher.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestClientCredentialTimeBasedTokenRefresher.java @@ -24,7 +24,7 @@ import org.apache.hadoop.util.Timer; import org.apache.http.HttpStatus; import org.junit.Test; -import org.mockserver.client.server.MockServerClient; +import org.mockserver.client.MockServerClient; import org.mockserver.integration.ClientAndServer; import org.mockserver.model.Header; import org.mockserver.model.HttpRequest; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestRefreshTokenTimeBasedTokenRefresher.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestRefreshTokenTimeBasedTokenRefresher.java index d9996bc4f2f0e..3ef105ca246c4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestRefreshTokenTimeBasedTokenRefresher.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestRefreshTokenTimeBasedTokenRefresher.java @@ -23,7 +23,7 @@ import org.apache.hadoop.util.Timer; import org.apache.http.HttpStatus; import org.junit.Test; -import org.mockserver.client.server.MockServerClient; +import org.mockserver.client.MockServerClient; import org.mockserver.integration.ClientAndServer; import org.mockserver.model.Header; import org.mockserver.model.HttpRequest; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index b0d7c4b14e1eb..8580547eb036d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -72,8 +72,8 @@ javax.servlet-api - com.google.guava - guava + org.apache.hadoop.thirdparty + hadoop-shaded-guava compile @@ -118,10 +118,6 @@ org.eclipse.jetty servlet-api-2.5 - - com.amazonaws - aws-java-sdk-s3 - org.eclipse.jdt core @@ -153,10 +149,6 @@ org.eclipse.jetty servlet-api-2.5 - - com.amazonaws - aws-java-sdk-s3 - org.eclipse.jdt core diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index a9ef653345b79..2866044badcd2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -23,7 +23,7 @@ import java.util.EnumSet; import java.util.List; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonPathCapabilities; @@ -54,6 +54,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.web.JsonUtilClient; import org.apache.hadoop.lib.wsrs.EnumSetParam; import org.apache.hadoop.security.UserGroupInformation; @@ -71,9 +72,9 @@ import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; @@ -198,6 +199,7 @@ public static FILE_TYPE getType(FileStatus fileStatus) { public static final String XATTR_VALUE_JSON = "value"; public static final String XATTRNAMES_JSON = "XAttrNames"; public static final String ECPOLICY_JSON = "ecPolicyObj"; + public static final String SYMLINK_JSON = "symlink"; public static final String FILE_CHECKSUM_JSON = "FileChecksum"; public static final String CHECKSUM_ALGORITHM_JSON = "algorithm"; @@ -262,7 +264,8 @@ public enum Operation { ALLOWSNAPSHOT(HTTP_PUT), DISALLOWSNAPSHOT(HTTP_PUT), CREATESNAPSHOT(HTTP_PUT), DELETESNAPSHOT(HTTP_DELETE), RENAMESNAPSHOT(HTTP_PUT), GETSNAPSHOTDIFF(HTTP_GET), - GETSNAPSHOTTABLEDIRECTORYLIST(HTTP_GET), GETSERVERDEFAULTS(HTTP_GET), + GETSNAPSHOTTABLEDIRECTORYLIST(HTTP_GET), GETSNAPSHOTLIST(HTTP_GET), + GETSERVERDEFAULTS(HTTP_GET), CHECKACCESS(HTTP_GET), SETECPOLICY(HTTP_PUT), GETECPOLICY( HTTP_GET), UNSETECPOLICY(HTTP_POST), SATISFYSTORAGEPOLICY(HTTP_PUT); @@ -1099,6 +1102,9 @@ private FileStatus createFileStatus(Path parent, JSONObject json) { String pathSuffix = (String) json.get(PATH_SUFFIX_JSON); Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix); FILE_TYPE type = FILE_TYPE.valueOf((String) json.get(TYPE_JSON)); + String symLinkValue = + type == FILE_TYPE.SYMLINK ? (String) json.get(SYMLINK_JSON) : null; + Path symLink = symLinkValue == null ? null : new Path(symLinkValue); long len = (Long) json.get(LENGTH_JSON); String owner = (String) json.get(OWNER_JSON); String group = (String) json.get(GROUP_JSON); @@ -1123,11 +1129,12 @@ private FileStatus createFileStatus(Path parent, JSONObject json) { new FsPermissionExtension(permission, aBit, eBit, ecBit); FileStatus fileStatus = new FileStatus(len, FILE_TYPE.DIRECTORY == type, replication, blockSize, mTime, aTime, deprecatedPerm, owner, group, - null, path, FileStatus.attributes(aBit, eBit, ecBit, seBit)); + symLink, path, FileStatus.attributes(aBit, eBit, ecBit, seBit)); return fileStatus; } else { return new FileStatus(len, FILE_TYPE.DIRECTORY == type, - replication, blockSize, mTime, aTime, permission, owner, group, path); + replication, blockSize, mTime, aTime, permission, owner, group, + symLink, path); } } @@ -1582,6 +1589,18 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirectoryList() return JsonUtilClient.toSnapshottableDirectoryList(json); } + public SnapshotStatus[] getSnapshotListing(Path snapshotRoot) + throws IOException { + Map params = new HashMap(); + params.put(OP_PARAM, Operation.GETSNAPSHOTLIST.toString()); + HttpURLConnection conn = getConnection( + Operation.GETSNAPSHOTLIST.getMethod(), + params, snapshotRoot, true); + HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); + return JsonUtilClient.toSnapshotList(json); + } + /** * This filesystem's capabilities must be in sync with that of * {@code DistributedFileSystem.hasPathCapability()} except diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index 0f8ea071e2d3f..67d4761543bcd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.XAttrCodec; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.http.client.HttpFSFileSystem; +import org.apache.hadoop.fs.http.client.HttpFSFileSystem.FILE_TYPE; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; @@ -44,6 +45,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.web.JsonUtil; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.lib.service.FileSystemAccess; @@ -110,8 +112,17 @@ private static Map toJsonInner(FileStatus fileStatus, Map json = new LinkedHashMap(); json.put(HttpFSFileSystem.PATH_SUFFIX_JSON, (emptyPathSuffix) ? "" : fileStatus.getPath().getName()); - json.put(HttpFSFileSystem.TYPE_JSON, - HttpFSFileSystem.FILE_TYPE.getType(fileStatus).toString()); + FILE_TYPE fileType = HttpFSFileSystem.FILE_TYPE.getType(fileStatus); + json.put(HttpFSFileSystem.TYPE_JSON, fileType.toString()); + if (fileType.equals(FILE_TYPE.SYMLINK)) { + // put the symlink into Json + try { + json.put(HttpFSFileSystem.SYMLINK_JSON, + fileStatus.getSymlink().getName()); + } catch (IOException e) { + // Can't happen. + } + } json.put(HttpFSFileSystem.LENGTH_JSON, fileStatus.getLen()); json.put(HttpFSFileSystem.OWNER_JSON, fileStatus.getOwner()); json.put(HttpFSFileSystem.GROUP_JSON, fileStatus.getGroup()); @@ -1834,6 +1845,43 @@ public String execute(FileSystem fs) throws IOException { } } + /** + * Executor that performs a getSnapshotListing operation. + */ + @InterfaceAudience.Private + public static class FSGetSnapshotListing implements + FileSystemAccess.FileSystemExecutor { + private Path path; + + /** + * Creates a getSnapshotDiff executor. + * @param path directory path of the snapshots to be examined. + */ + public FSGetSnapshotListing(String path) { + this.path = new Path(path); + } + + /** + * Executes the filesystem operation. + * @param fs filesystem instance to use. + * @return A JSON string of all snapshots for a snapshottable directory. + * @throws IOException thrown if an IO error occurred. + */ + @Override + public String execute(FileSystem fs) throws IOException { + SnapshotStatus[] sds = null; + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem dfs = (DistributedFileSystem) fs; + sds = dfs.getSnapshotListing(path); + } else { + throw new UnsupportedOperationException("getSnapshotListing is " + + "not supported for HttpFs on " + fs.getClass() + + ". Please check your fs.defaultFS configuration"); + } + return JsonUtil.toJsonString(sds); + } + } + /** * Executor that performs a getServerDefaults operation. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java index 39180fbe4ae8a..f6c84dcae4e07 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java @@ -116,6 +116,7 @@ public class HttpFSParametersProvider extends ParametersProvider { new Class[] {OldSnapshotNameParam.class, SnapshotNameParam.class}); PARAMS_DEF.put(Operation.GETSNAPSHOTTABLEDIRECTORYLIST, new Class[] {}); + PARAMS_DEF.put(Operation.GETSNAPSHOTLIST, new Class[] {}); PARAMS_DEF.put(Operation.GETSERVERDEFAULTS, new Class[] {}); PARAMS_DEF.put(Operation.CHECKACCESS, new Class[] {FsActionParam.class}); PARAMS_DEF.put(Operation.SETECPOLICY, new Class[] {ECPolicyParam.class}); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java index bae9dd19b4053..0e7038b635d35 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.http.server; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -107,8 +107,38 @@ @Path(HttpFSFileSystem.SERVICE_VERSION) @InterfaceAudience.Private public class HttpFSServer { + + enum AccessMode { + READWRITE, WRITEONLY, READONLY; + } private static Logger AUDIT_LOG = LoggerFactory.getLogger("httpfsaudit"); private static final Logger LOG = LoggerFactory.getLogger(HttpFSServer.class); + AccessMode accessMode = AccessMode.READWRITE; + + public HttpFSServer() { + Configuration conf = HttpFSServerWebApp.get().getConfig(); + final String accessModeString = conf.get("httpfs.access.mode", "read-write").toLowerCase(); + if(accessModeString.compareTo("write-only") == 0) + accessMode = AccessMode.WRITEONLY; + else if(accessModeString.compareTo("read-only") == 0) + accessMode = AccessMode.READONLY; + else + accessMode = AccessMode.READWRITE; + } + + + // First try getting a user through HttpUserGroupInformation. This will return + // if the built-in hadoop auth filter is not used. Fall back to getting the + // authenticated user from the request. + private UserGroupInformation getHttpUGI(HttpServletRequest request) { + UserGroupInformation user = HttpUserGroupInformation.get(); + if (user != null) { + return user; + } + + return UserGroupInformation.createRemoteUser(request.getUserPrincipal().getName()); + } + /** * Executes a {@link FileSystemAccess.FileSystemExecutor} using a filesystem for the effective @@ -219,6 +249,12 @@ public Response get(@PathParam("path") String path, @Context Parameters params, @Context HttpServletRequest request) throws IOException, FileSystemAccessException { + // Restrict access to only GETFILESTATUS and LISTSTATUS in write-only mode + if((op.value() != HttpFSFileSystem.Operation.GETFILESTATUS) && + (op.value() != HttpFSFileSystem.Operation.LISTSTATUS) && + accessMode == AccessMode.WRITEONLY) { + return Response.status(Response.Status.FORBIDDEN).build(); + } UserGroupInformation user = HttpUserGroupInformation.get(); Response response; path = makeAbsolute(path); @@ -422,6 +458,14 @@ public InputStream run() throws Exception { response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); break; } + case GETSNAPSHOTLIST: { + FSOperations.FSGetSnapshotListing command = + new FSOperations.FSGetSnapshotListing(path); + String js = fsExecute(user, command); + AUDIT_LOG.info("[{}]", "/"); + response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + break; + } case GETSERVERDEFAULTS: { FSOperations.FSGetServerDefaults command = new FSOperations.FSGetServerDefaults(); @@ -491,6 +535,10 @@ public Response delete(@PathParam("path") String path, @Context Parameters params, @Context HttpServletRequest request) throws IOException, FileSystemAccessException { + // Do not allow DELETE commands in read-only mode + if(accessMode == AccessMode.READONLY) { + return Response.status(Response.Status.FORBIDDEN).build(); + } UserGroupInformation user = HttpUserGroupInformation.get(); Response response; path = makeAbsolute(path); @@ -578,6 +626,10 @@ public Response post(InputStream is, @Context Parameters params, @Context HttpServletRequest request) throws IOException, FileSystemAccessException { + // Do not allow POST commands in read-only mode + if(accessMode == AccessMode.READONLY) { + return Response.status(Response.Status.FORBIDDEN).build(); + } UserGroupInformation user = HttpUserGroupInformation.get(); Response response; path = makeAbsolute(path); @@ -718,6 +770,10 @@ public Response put(InputStream is, @Context Parameters params, @Context HttpServletRequest request) throws IOException, FileSystemAccessException { + // Do not allow PUT commands in read-only mode + if(accessMode == AccessMode.READONLY) { + return Response.status(Response.Status.FORBIDDEN).build(); + } UserGroupInformation user = HttpUserGroupInformation.get(); Response response; path = makeAbsolute(path); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java index 5e855de7eec85..985feed09981c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java @@ -18,7 +18,7 @@ package org.apache.hadoop.lib.servlet; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.lib.server.Server; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java index e0f62002c70d4..c171e929ca6d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java @@ -19,7 +19,7 @@ import org.apache.hadoop.classification.InterfaceAudience; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import java.util.List; import java.util.Map; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ParametersProvider.java index c93f8f2b17db1..56a999f519cdf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ParametersProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ParametersProvider.java @@ -18,7 +18,7 @@ package org.apache.hadoop.lib.wsrs; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import com.sun.jersey.api.core.HttpContext; import com.sun.jersey.core.spi.component.ComponentContext; import com.sun.jersey.core.spi.component.ComponentScope; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml index 5b8e469e96bc7..50a16af630871 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml @@ -323,4 +323,15 @@ + + httpfs.access.mode + read-write + + Sets the access mode for HTTPFS. If access is not allowed the FORBIDDED (403) is returned. + Valid access modes are: + read-write Full Access allowed + write-only PUT POST and DELETE full Access. GET only allows GETFILESTATUS and LISTSTATUS + read-only GET Full Access PUT POST and DELETE are FORBIDDEN + + diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java index 106e475897349..a53d17fc2d2f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.web.JsonUtil; @@ -74,7 +75,7 @@ import org.eclipse.jetty.server.Server; import org.eclipse.jetty.webapp.WebAppContext; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import java.io.File; import java.io.FileOutputStream; @@ -365,6 +366,42 @@ private void testDelete() throws Exception { fs.close(); } + private void testListSymLinkStatus() throws Exception { + if (isLocalFS()) { + // do not test the the symlink for local FS. + return; + } + FileSystem fs = FileSystem.get(getProxiedFSConf()); + boolean isWebhdfs = fs instanceof WebHdfsFileSystem; + Path path = + new Path(getProxiedFSTestDir() + "-symlink", "targetFoo.txt"); + OutputStream os = fs.create(path); + os.write(1); + os.close(); + Path linkPath = + new Path(getProxiedFSTestDir()+ "-symlink", "symlinkFoo.txt"); + fs.createSymlink(path, linkPath, false); + fs = getHttpFSFileSystem(); + FileStatus linkStatus = fs.getFileStatus(linkPath); + FileStatus status1 = fs.getFileStatus(path); + + FileStatus[] stati = fs.listStatus(path.getParent()); + assertEquals(2, stati.length); + + int countSymlink = 0; + for (int i = 0; i < stati.length; i++) { + FileStatus fStatus = stati[i]; + countSymlink += fStatus.isSymlink() ? 1 : 0; + } + assertEquals(1, countSymlink); + + assertFalse(status1.isSymlink()); + if (isWebhdfs) { + assertTrue(linkStatus.isSymlink()); + } + fs.close(); + } + private void testListStatus() throws Exception { FileSystem fs = FileSystem.get(getProxiedFSConf()); boolean isDFS = fs instanceof DistributedFileSystem; @@ -1158,7 +1195,8 @@ protected enum Operation { CREATE_SNAPSHOT, RENAME_SNAPSHOT, DELETE_SNAPSHOT, ALLOW_SNAPSHOT, DISALLOW_SNAPSHOT, DISALLOW_SNAPSHOT_EXCEPTION, FILE_STATUS_ATTR, GET_SNAPSHOT_DIFF, GET_SNAPSHOTTABLE_DIRECTORY_LIST, - GET_SERVERDEFAULTS, CHECKACCESS, SETECPOLICY, SATISFYSTORAGEPOLICY + GET_SNAPSHOT_LIST, GET_SERVERDEFAULTS, CHECKACCESS, SETECPOLICY, + SATISFYSTORAGEPOLICY } private void operation(Operation op) throws Exception { @@ -1189,6 +1227,7 @@ private void operation(Operation op) throws Exception { break; case LIST_STATUS: testListStatus(); + testListSymLinkStatus(); break; case WORKING_DIRECTORY: testWorkingdirectory(); @@ -1279,6 +1318,9 @@ private void operation(Operation op) throws Exception { case GET_SNAPSHOTTABLE_DIRECTORY_LIST: testGetSnapshottableDirListing(); break; + case GET_SNAPSHOT_LIST: + testGetSnapshotListing(); + break; case GET_SERVERDEFAULTS: testGetServerDefaults(); break; @@ -1657,6 +1699,50 @@ private void verifyGetSnapshottableDirListing( JsonUtil.toJsonString(dfssds)); } + private void testGetSnapshotListing() throws Exception { + if (!this.isLocalFS()) { + // Create a directory with snapshot allowed + Path path = new Path("/tmp/tmp-snap-test"); + createSnapshotTestsPreconditions(path); + // Get the FileSystem instance that's being tested + FileSystem fs = this.getHttpFSFileSystem(); + // Check FileStatus + Assert.assertTrue(fs.getFileStatus(path).isSnapshotEnabled()); + // Create a file and take a snapshot + Path file1 = new Path(path, "file1"); + testCreate(file1, false); + fs.createSnapshot(path, "snap1"); + // Create another file and take a snapshot + Path file2 = new Path(path, "file2"); + testCreate(file2, false); + fs.createSnapshot(path, "snap2"); + // Get snapshot diff + SnapshotStatus[] snapshotStatus = null; + if (fs instanceof HttpFSFileSystem) { + HttpFSFileSystem httpFS = (HttpFSFileSystem) fs; + snapshotStatus = httpFS.getSnapshotListing(path); + } else if (fs instanceof WebHdfsFileSystem) { + WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs; + snapshotStatus = webHdfsFileSystem.getSnapshotListing(path); + } else { + Assert.fail(fs.getClass().getSimpleName() + + " doesn't support getSnapshotDiff"); + } + // Verify result with DFS + DistributedFileSystem dfs = (DistributedFileSystem) + FileSystem.get(path.toUri(), this.getProxiedFSConf()); + SnapshotStatus[] dfsStatus = + dfs.getSnapshotListing(path); + Assert.assertEquals(JsonUtil.toJsonString(snapshotStatus), + JsonUtil.toJsonString(dfsStatus)); + // Cleanup + fs.deleteSnapshot(path, "snap2"); + fs.deleteSnapshot(path, "snap1"); + fs.delete(path, true); + } + } + + private void testGetSnapshottableDirListing() throws Exception { if (!this.isLocalFS()) { FileSystem fs = this.getHttpFSFileSystem(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSAccessControlled.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSAccessControlled.java new file mode 100644 index 0000000000000..47d99352de1b2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSAccessControlled.java @@ -0,0 +1,355 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.http.server; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.test.HTestCase; +import org.apache.hadoop.test.HadoopUsersConfTestHelper; +import org.apache.hadoop.test.TestDir; +import org.apache.hadoop.test.TestDirHelper; +import org.apache.hadoop.test.TestJetty; +import org.apache.hadoop.test.TestJettyHelper; +import org.junit.Assert; +import org.junit.Test; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.webapp.WebAppContext; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.Writer; +import java.net.HttpURLConnection; +import java.net.URL; +import java.text.MessageFormat; + +/** + * This test class ensures that everything works as expected when + * support with the access controlled HTTPFS file system. + */ +public class TestHttpFSAccessControlled extends HTestCase { + + private MiniDFSCluster miniDfs; + private Configuration nnConf; + + /** + * Fire up our own hand-rolled MiniDFSCluster. We do this here instead + * of relying on TestHdfsHelper because we don't want to turn on ACL + * support. + * + * @throws Exception + */ + private void startMiniDFS() throws Exception { + + File testDirRoot = TestDirHelper.getTestDir(); + + if (System.getProperty("hadoop.log.dir") == null) { + System.setProperty("hadoop." + + "log.dir", + new File(testDirRoot, "hadoop-log").getAbsolutePath()); + } + if (System.getProperty("test.build.data") == null) { + System.setProperty("test.build.data", + new File(testDirRoot, "hadoop-data").getAbsolutePath()); + } + + Configuration conf = HadoopUsersConfTestHelper.getBaseConf(); + HadoopUsersConfTestHelper.addUserConf(conf); + conf.set("fs.hdfs.impl.disable.cache", "true"); + conf.set("dfs.block.access.token.enable", "false"); + conf.set("dfs.permissions", "true"); + conf.set("hadoop.security.authentication", "simple"); + + MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); + builder.numDataNodes(2); + miniDfs = builder.build(); + nnConf = miniDfs.getConfiguration(0); + } + + /** + * Create an HttpFS Server to talk to the MiniDFSCluster we created. + * @throws Exception + */ + private void createHttpFSServer() throws Exception { + File homeDir = TestDirHelper.getTestDir(); + Assert.assertTrue(new File(homeDir, "conf").mkdir()); + Assert.assertTrue(new File(homeDir, "log").mkdir()); + Assert.assertTrue(new File(homeDir, "temp").mkdir()); + HttpFSServerWebApp.setHomeDirForCurrentThread(homeDir.getAbsolutePath()); + + File secretFile = new File(new File(homeDir, "conf"), "secret"); + Writer w = new FileWriter(secretFile); + w.write("secret"); + w.close(); + + // HDFS configuration + File hadoopConfDir = new File(new File(homeDir, "conf"), "hadoop-conf"); + if ( !hadoopConfDir.mkdirs() ) { + throw new IOException(); + } + + String fsDefaultName = + nnConf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY); + Configuration conf = new Configuration(false); + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsDefaultName); + + File hdfsSite = new File(hadoopConfDir, "hdfs-site.xml"); + OutputStream os = new FileOutputStream(hdfsSite); + conf.writeXml(os); + os.close(); + + // HTTPFS configuration + conf = new Configuration(false); + conf.set("httpfs.hadoop.config.dir", hadoopConfDir.toString()); + conf.set("httpfs.proxyuser." + + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".groups", + HadoopUsersConfTestHelper.getHadoopProxyUserGroups()); + conf.set("httpfs.proxyuser." + + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".hosts", + HadoopUsersConfTestHelper.getHadoopProxyUserHosts()); + conf.set("httpfs.authentication.signature.secret.file", + secretFile.getAbsolutePath()); + + File httpfsSite = new File(new File(homeDir, "conf"), "httpfs-site.xml"); + os = new FileOutputStream(httpfsSite); + conf.writeXml(os); + os.close(); + + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + URL url = cl.getResource("webapp"); + if ( url == null ) { + throw new IOException(); + } + WebAppContext context = new WebAppContext(url.getPath(), "/webhdfs"); + Server server = TestJettyHelper.getJettyServer(); + server.setHandler(context); + server.start(); + } + + /** + * Talks to the http interface to get the json output of a *STATUS command + * on the given file. + * + * @param filename The file to query. + * @param message Failure message + * @param command Command to test + * @param expectOK Is this operation expected to succeed? + * @throws Exception + */ + private void getCmd(String filename, String message, String command, boolean expectOK) + throws Exception { + String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; + String outMsg = message + " (" + command + ")"; + // Remove leading / from filename + if ( filename.charAt(0) == '/' ) { + filename = filename.substring(1); + } + String pathOps = MessageFormat.format( + "/webhdfs/v1/{0}?user.name={1}&op={2}", + filename, user, command); + URL url = new URL(TestJettyHelper.getJettyURL(), pathOps); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + conn.connect(); + int resp = conn.getResponseCode(); + if ( expectOK ) { + Assert.assertEquals( outMsg, HttpURLConnection.HTTP_OK, resp); + } else { + Assert.assertEquals(outMsg, HttpURLConnection.HTTP_FORBIDDEN, resp); + } + } + + /** + * General-purpose http PUT command to the httpfs server. + * @param filename The file to operate upon + * @param message Failure message + * @param command The command to perform (SETPERMISSION, etc) + * @param params Parameters to command + * @param expectOK Is this operation expected to succeed? + */ + private void putCmd(String filename, String message, String command, + String params, boolean expectOK) throws Exception { + String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; + String outMsg = message + " (" + command + ")"; + // Remove leading / from filename + if ( filename.charAt(0) == '/' ) { + filename = filename.substring(1); + } + String pathOps = MessageFormat.format( + "/webhdfs/v1/{0}?user.name={1}{2}{3}&op={4}", + filename, user, (params == null) ? "" : "&", + (params == null) ? "" : params, command); + URL url = new URL(TestJettyHelper.getJettyURL(), pathOps); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + conn.connect(); + int resp = conn.getResponseCode(); + if ( expectOK ) { + Assert.assertEquals(outMsg, HttpURLConnection.HTTP_OK, resp); + } else { + Assert.assertEquals(outMsg, HttpURLConnection.HTTP_FORBIDDEN, resp); + } + } + + /** + * General-purpose http PUT command to the httpfs server. + * @param filename The file to operate upon + * @param message Failure message + * @param command The command to perform (SETPERMISSION, etc) + * @param params Parameters to command + * @param expectOK Is this operation expected to succeed? + */ + private void deleteCmd(String filename, String message, String command, + String params, boolean expectOK) throws Exception { + String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; + String outMsg = message + " (" + command + ")"; + // Remove leading / from filename + if ( filename.charAt(0) == '/' ) { + filename = filename.substring(1); + } + String pathOps = MessageFormat.format( + "/webhdfs/v1/{0}?user.name={1}{2}{3}&op={4}", + filename, user, (params == null) ? "" : "&", + (params == null) ? "" : params, command); + URL url = new URL(TestJettyHelper.getJettyURL(), pathOps); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("DELETE"); + conn.connect(); + int resp = conn.getResponseCode(); + if ( expectOK ) { + Assert.assertEquals(outMsg, HttpURLConnection.HTTP_OK, resp); + } else { + Assert.assertEquals(outMsg, HttpURLConnection.HTTP_FORBIDDEN, resp); + } + } + + /** + * General-purpose http POST command to the httpfs server. + * @param filename The file to operate upon + * @param message Failure message + * @param command The command to perform (UNSETSTORAGEPOLICY, etc) + * @param params Parameters to command" + * @param expectOK Is this operation expected to succeed? + */ + private void postCmd(String filename, String message, String command, + String params, boolean expectOK) throws Exception { + String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; + String outMsg = message + " (" + command + ")"; + // Remove leading / from filename + if ( filename.charAt(0) == '/' ) { + filename = filename.substring(1); + } + String pathOps = MessageFormat.format( + "/webhdfs/v1/{0}?user.name={1}{2}{3}&op={4}", + filename, user, (params == null) ? "" : "&", + (params == null) ? "" : params, command); + URL url = new URL(TestJettyHelper.getJettyURL(), pathOps); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.connect(); + int resp = conn.getResponseCode(); + if ( expectOK ) { + Assert.assertEquals(outMsg, HttpURLConnection.HTTP_OK, resp); + } else { + Assert.assertEquals(outMsg, HttpURLConnection.HTTP_FORBIDDEN, resp); + } + } + + /** + * Ensure that + *

    + *
  1. GETFILESTATUS (GET) and LISTSTATUS (GET) work in all modes
    2. + *
  2. GETXATTRS (GET) works in read-write and read-only but write-only throws an exception
    3. + *
  3. SETPERMISSION (PUT) works in read-write and write only but read-only throws an exception
    4. + *
  4. SETPERMISSION (POST) works in read-write and write only but read-only throws an exception
    5. + *
  5. DELETE (DELETE) works in read-write and write only but read-only throws an exception
    6. + *
+ * + * @throws Exception + */ + @Test + @TestDir + @TestJetty + public void testAcessControlledFS() throws Exception { + final String testRwMsg = "Test read-write "; + final String testRoMsg = "Test read-only "; + final String testWoMsg = "Test write-only "; + final String defUser1 = "default:user:glarch:r-x"; + final String dir = "/testAccess"; + final String pathRW = dir + "/foo-rw"; + final String pathWO = dir + "/foo-wo"; + final String pathRO = dir + "/foo-ro"; + final String setPermSpec = "744"; + final String snapshopSpec = "snapshotname=test-snap"; + startMiniDFS(); + createHttpFSServer(); + + FileSystem fs = FileSystem.get(nnConf); + fs.mkdirs(new Path(dir)); + OutputStream os = fs.create(new Path(pathRW)); + os.write(1); + os.close(); + + os = fs.create(new Path(pathWO)); + os.write(1); + os.close(); + + os = fs.create(new Path(pathRO)); + os.write(1); + os.close(); + + Configuration conf = HttpFSServerWebApp.get().getConfig(); + + /* test Read-Write Mode */ + conf.setStrings("httpfs.access.mode", "read-write"); + getCmd(pathRW, testRwMsg + "GET", "GETFILESTATUS", true); + getCmd(pathRW, testRwMsg + "GET", "LISTSTATUS", true); + getCmd(pathRW, testRwMsg + "GET", "GETXATTRS", true); + putCmd(pathRW, testRwMsg + "PUT", "SETPERMISSION", setPermSpec, true); + postCmd(pathRW, testRwMsg + "POST", "UNSETSTORAGEPOLICY", null, true); + deleteCmd(pathRW, testRwMsg + "DELETE", "DELETE", null, true); + + /* test Write-Only Mode */ + conf.setStrings("httpfs.access.mode", "write-only"); + getCmd(pathWO, testWoMsg + "GET", "GETFILESTATUS", true); + getCmd(pathWO, testWoMsg + "GET", "LISTSTATUS", true); + getCmd(pathWO, testWoMsg + "GET", "GETXATTRS", false); + putCmd(pathWO, testWoMsg + "PUT", "SETPERMISSION", setPermSpec, true); + postCmd(pathWO, testWoMsg + "POST", "UNSETSTORAGEPOLICY", null, true); + deleteCmd(pathWO, testWoMsg + "DELETE", "DELETE", null, true); + + /* test Read-Only Mode */ + conf.setStrings("httpfs.access.mode", "read-only"); + getCmd(pathRO, testRoMsg + "GET", "GETFILESTATUS", true); + getCmd(pathRO, testRoMsg + "GET", "LISTSTATUS", true); + getCmd(pathRO, testRoMsg + "GET", "GETXATTRS", true); + putCmd(pathRO, testRoMsg + "PUT", "SETPERMISSION", setPermSpec, false); + postCmd(pathRO, testRoMsg + "POST", "UNSETSTORAGEPOLICY", null, false); + deleteCmd(pathRO, testRoMsg + "DELETE", "DELETE", null, false); + + conf.setStrings("httpfs.access.mode", "read-write"); + + miniDfs.shutdown(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java index 6739393924e5e..434f53cf5237d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.web.JsonUtil; @@ -105,7 +106,7 @@ import org.eclipse.jetty.server.Server; import org.eclipse.jetty.webapp.WebAppContext; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import java.util.Properties; import java.util.regex.Pattern; @@ -1515,6 +1516,23 @@ private void verifyGetSnapshottableDirectoryList(DistributedFileSystem dfs) Assert.assertEquals(dirLst, JsonUtil.toJsonString(dfsDirLst)); } + private void verifyGetSnapshotList(DistributedFileSystem dfs, Path path) + throws Exception { + // Send a request + HttpURLConnection conn = sendRequestToHttpFSServer(path.toString(), + "GETSNAPSHOTLIST", ""); + // Should return HTTP_OK + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + // Verify the response + BufferedReader reader = + new BufferedReader(new InputStreamReader(conn.getInputStream())); + // The response should be a one-line JSON string. + String dirLst = reader.readLine(); + // Verify the content of status with DFS API. + SnapshotStatus[] dfsDirLst = dfs.getSnapshotListing(path); + Assert.assertEquals(dirLst, JsonUtil.toJsonString(dfsDirLst)); + } + @Test @TestDir @TestJetty @@ -1550,6 +1568,35 @@ public void testGetSnapshottableDirectoryList() throws Exception { verifyGetSnapshottableDirectoryList(dfs); } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testGetSnapshotList() throws Exception { + createHttpFSServer(false, false); + // Create test directories + String pathStr = "/tmp/tmp-snap-list-test-1"; + createDirWithHttp(pathStr, "700", null); + Path path = new Path(pathStr); + DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get( + path.toUri(), TestHdfsHelper.getHdfsConf()); + // Enable snapshot for path1 + dfs.allowSnapshot(path); + Assert.assertTrue(dfs.getFileStatus(path).isSnapshotEnabled()); + // Verify response when there is one snapshottable directory + verifyGetSnapshotList(dfs, path); + // Create a file and take a snapshot + String file1 = pathStr + "/file1"; + createWithHttp(file1, null); + dfs.createSnapshot(path, "snap1"); + // Create another file and take a snapshot + String file2 = pathStr + "/file2"; + createWithHttp(file2, null); + dfs.createSnapshot(path, "snap2"); + verifyGetSnapshotList(dfs, path); + } + @Test @TestDir @TestJetty diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/DummyGroupMapping.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/DummyGroupMapping.java index 2693deff7d93a..80a94b18d1e51 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/DummyGroupMapping.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/DummyGroupMapping.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.Set; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import org.apache.hadoop.security.GroupMappingServiceProvider; import org.apache.hadoop.test.HadoopUsersConfTestHelper; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt index 6e233fd3991d6..6b8a795204639 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt @@ -153,7 +153,8 @@ add_subdirectory(main/native/libhdfs-examples) # Temporary fix to disable Libhdfs++ build on older systems that do not support thread_local include(CheckCXXSourceCompiles) unset (THREAD_LOCAL_SUPPORTED CACHE) -set (CMAKE_REQUIRED_DEFINITIONS "-std=c++11") +set (CMAKE_CXX_STANDARD 11) +set (CMAKE_CXX_STANDARD_REQUIRED ON) set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) check_cxx_source_compiles( "#include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c index 9054287405632..846852bfd0e88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c @@ -279,7 +279,7 @@ static int testHdfsMiniStressImpl(struct tlhThreadInfo *ti) EXPECT_NONNULL(ti->hdfs); // Error injection on, some failures are expected in the read path. // The expectation is that any memory stomps will cascade and cause - // the following test to fail. Ideally RPC errors would be seperated + // the following test to fail. Ideally RPC errors would be separated // from BlockReader errors (RPC is expected to recover from disconnects). doTestHdfsMiniStress(ti, 1); // No error injection diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt index 6a2f378d0a4bb..914fefdafce6b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt @@ -53,7 +53,8 @@ include(CheckCXXSourceCompiles) # Check if thread_local is supported unset (THREAD_LOCAL_SUPPORTED CACHE) -set (CMAKE_REQUIRED_DEFINITIONS "-std=c++11") +set (CMAKE_CXX_STANDARD 11) +set (CMAKE_CXX_STANDARD_REQUIRED ON) set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) check_cxx_source_compiles( "#include @@ -149,12 +150,13 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0") set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0") if(UNIX) -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -std=c++11 -g -fPIC -fno-strict-aliasing") +set (CMAKE_CXX_STANDARD 11) +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -g -fPIC -fno-strict-aliasing") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fPIC -fno-strict-aliasing") endif() if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + set(CMAKE_CXX_STANDARD 11) add_definitions(-DASIO_HAS_STD_ADDRESSOF -DASIO_HAS_STD_ARRAY -DASIO_HAS_STD_ATOMIC -DASIO_HAS_CSTDINT -DASIO_HAS_STD_SHARED_PTR -DASIO_HAS_STD_TYPE_TRAITS -DASIO_HAS_VARIADIC_TEMPLATES -DASIO_HAS_STD_FUNCTION -DASIO_HAS_STD_CHRONO -DASIO_HAS_STD_SYSTEM_ERROR) endif () @@ -265,6 +267,7 @@ if (HADOOP_BUILD) ${CMAKE_THREAD_LIBS_INIT} ) set_target_properties(hdfspp PROPERTIES SOVERSION ${LIBHDFSPP_VERSION}) + hadoop_dual_output_directory(hdfspp ${OUT_DIR}) else (HADOOP_BUILD) add_library(hdfspp_static STATIC ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS}) target_link_libraries(hdfspp_static diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h index d8574d15b3195..bc3d8b96d3f66 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h @@ -103,7 +103,7 @@ class URI { std::string str(bool encoded_output=true) const; - // Get a string with each URI field printed on a seperate line + // Get a string with each URI field printed on a separate line std::string GetDebugString() const; private: // These are stored in encoded form diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/uriparser2/uriparser/UriFile.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/uriparser2/uriparser/UriFile.c index 5471e5af8ccb2..22f38bee25bd2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/uriparser2/uriparser/UriFile.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/uriparser2/uriparser/UriFile.c @@ -90,7 +90,7 @@ static URI_INLINE int URI_FUNC(FilenameToUriString)(const URI_CHAR * filename, if ((input[0] == _UT('\0')) || (fromUnix && input[0] == _UT('/')) || (!fromUnix && input[0] == _UT('\\'))) { - /* Copy text after last seperator */ + /* Copy text after last separator */ if (lastSep + 1 < input) { if (!fromUnix && absolute && (firstSegment == URI_TRUE)) { /* Quick hack to not convert "C:" to "C%3A" */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml index e95f994e3c2c8..18a91cbcdc6f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml @@ -84,8 +84,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> test - com.google.guava - guava + org.apache.hadoop.thirdparty + hadoop-shaded-guava compile diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java index 27213953802f7..3b0327ad4a149 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java @@ -55,7 +55,7 @@ import org.jboss.netty.buffer.ChannelBuffers; import org.jboss.netty.channel.ChannelHandlerContext; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * RPC program corresponding to mountd daemon. See {@link Mountd}. diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java index e0fb302992850..41add2212936a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java @@ -29,7 +29,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSClient; @@ -40,13 +40,13 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ShutdownHookManager; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Objects; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Objects; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; +import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; +import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalListener; +import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalNotification; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java index 8494493beed70..ff64ad5804609 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java @@ -26,7 +26,7 @@ import org.apache.hadoop.nfs.nfs3.Nfs3Base; import org.apache.hadoop.util.StringUtils; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Nfs server. Supports NFS v3 using {@link RpcProgramNfs3}. diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OffsetRange.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OffsetRange.java index 764524a8ff677..3995fa5566bb0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OffsetRange.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OffsetRange.java @@ -19,7 +19,7 @@ import java.util.Comparator; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * OffsetRange is the range of read/write request. A single point (e.g.,[5,5]) diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java index 6067a5df34786..528ead7a003b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java @@ -57,8 +57,8 @@ import org.apache.hadoop.util.Time; import org.jboss.netty.channel.Channel; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java index 5c915d26bf1e0..b8db83c89a3a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java @@ -30,9 +30,9 @@ import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.Time; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; /** * A cache saves OpenFileCtx objects for different users. Each cache entry is diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java index cb46f449a1f00..d436eac598be0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java @@ -134,7 +134,7 @@ import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.ChannelHandlerContext; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java index 98f3d6cfa2930..76859247bf2a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java @@ -29,8 +29,8 @@ import org.apache.hadoop.nfs.nfs3.Nfs3Constant.WriteStableHow; import org.jboss.netty.channel.Channel; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * WriteCtx saves the context of one write request, such as request, channel, diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java index 35542391bdb4c..288937104084b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java @@ -45,7 +45,7 @@ import org.apache.hadoop.security.IdMappingServiceProvider; import org.jboss.netty.channel.Channel; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Manage the writes and responds asynchronously. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml index 782f28a0ef30e..23e0b8feb0df9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml @@ -114,6 +114,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> mockito-core test + + org.assertj + assertj-core + test + diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java index b16a6c0df56ba..dd92725592ec5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java @@ -50,9 +50,9 @@ public class FederationRPCMetrics implements FederationRPCMBean { @Metric("Number of operations the Router proxied to a Namenode") private MutableCounterLong proxyOp; - @Metric("Number of operations to fail to reach NN") - private MutableCounterLong proxyOpFailureStandby; @Metric("Number of operations to hit a standby NN") + private MutableCounterLong proxyOpFailureStandby; + @Metric("Number of operations to fail to reach NN") private MutableCounterLong proxyOpFailureCommunicate; @Metric("Number of operations to hit a client overloaded Router") private MutableCounterLong proxyOpFailureClientOverloaded; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java index 5f06f5918ea5f..64936e28a651e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java @@ -33,7 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import static org.apache.hadoop.util.Time.monotonicNow; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java index 39e7d9779c197..8596d9db24aba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java @@ -61,9 +61,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; +import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; /** * Expose the Namenode metrics as the Router was one. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java index e13815bb4ac3e..9e1fb67173e6c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java @@ -88,7 +88,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Implementation of the Router metrics collector. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java index 64bb10822f9f8..b4932219d39c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java @@ -32,7 +32,7 @@ import org.apache.hadoop.metrics2.lib.MutableGaugeInt; import org.apache.hadoop.metrics2.lib.MutableRate; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Implementations of the JMX interface for the State Store metrics. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java index 96b560c649a97..797006ab1de4a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java @@ -61,9 +61,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.cache.Cache; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; /** * Mount table to map between global paths and remote locations. This allows the diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MultipleDestinationMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MultipleDestinationMountTableResolver.java index b09a883ebcb77..4055432502058 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MultipleDestinationMountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MultipleDestinationMountTableResolver.java @@ -32,7 +32,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Mount table resolver that supports multiple locations for each mount entry. @@ -99,8 +99,9 @@ public PathLocation getDestinationForPath(String path) throws IOException { // Change the order of the name spaces according to the policy if (firstNamespace != null) { - // This is the entity in the tree, we need to create our own copy - mountTableResult = new PathLocation(mountTableResult, firstNamespace); + // Create our own prioritized copy based on the entity in the tree. + mountTableResult = PathLocation.prioritizeDestination( + mountTableResult, firstNamespace); LOG.debug("Ordered locations following {} are {}", order, mountTableResult); } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java index cb04ff8f9312c..44856cb5e8996 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java @@ -87,12 +87,28 @@ public PathLocation(final PathLocation other) { * @param other Other path location to copy from. * @param firstNsId Identifier of the namespace to place first. */ + @Deprecated public PathLocation(PathLocation other, String firstNsId) { this.sourcePath = other.sourcePath; this.destOrder = other.destOrder; this.destinations = orderedNamespaces(other.destinations, firstNsId); } + /** + * Return a path location with the prioritized destinations based on + * the current path location. + * + * @param base The base path location we'd like to prioritize on. + * @param firstNsId Identifier of the namespace to place first. + */ + public static PathLocation prioritizeDestination( + PathLocation base, String firstNsId) { + List prioritizedDestinations = orderedNamespaces( + base.destinations, firstNsId); + return new PathLocation(base.sourcePath, prioritizedDestinations, + base.destOrder); + } + /** * Prioritize a location/destination by its name space/nameserviceId. * This destination might be used by other threads, so the source is not diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/AvailableSpaceResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/AvailableSpaceResolver.java index 883a126236e6f..88e20649506dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/AvailableSpaceResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/AvailableSpaceResolver.java @@ -39,7 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Order the destinations based on available space. This resolver uses a diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashResolver.java index 455a3edb87d0b..3f8c354913c18 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashResolver.java @@ -29,7 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Order the destinations based on consistent hashing. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java index 58a8ed278b3dc..3da655e35d094 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java @@ -42,8 +42,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.net.HostAndPort; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.net.HostAndPort; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java index 13643e5e9f754..d21eef545b3b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java @@ -25,7 +25,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Iterables; +import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables; /** * Order the destinations randomly. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java index 74bbbb572fd27..9ec3b54ed50b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java @@ -32,7 +32,7 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java index 60924990cfd30..52e7cebd26017 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java @@ -31,7 +31,7 @@ import javax.net.SocketFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java index 7cb343b1d53e8..8b537f9d94c11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java @@ -23,7 +23,7 @@ import java.util.Collections; import java.util.List; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/MountTableRefresherService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/MountTableRefresherService.java index e998ab0a9dba4..0246ea98c29f2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/MountTableRefresherService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/MountTableRefresherService.java @@ -40,13 +40,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; +import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; +import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalListener; +import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalNotification; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; /** * This service is invoked from {@link MountTableStore} when there is change in diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java index 5e1222247286c..b690b8685c0a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java @@ -28,7 +28,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; /** * Service to periodically execute a runnable. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java index 2919ddc4c506d..9d84559212428 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java @@ -42,8 +42,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ListMultimap; +import org.apache.hadoop.thirdparty.com.google.common.collect.ArrayListMultimap; +import org.apache.hadoop.thirdparty.com.google.common.collect.ListMultimap; /** * Module that implements the quota relevant RPC calls diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java index 7b06ca428bd38..ba3659e0bdbff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java @@ -277,6 +277,10 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic { public static final String DFS_ROUTER_ADMIN_MAX_COMPONENT_LENGTH_KEY = FEDERATION_ROUTER_PREFIX + "fs-limits.max-component-length"; public static final int DFS_ROUTER_ADMIN_MAX_COMPONENT_LENGTH_DEFAULT = 0; + public static final String DFS_ROUTER_ADMIN_MOUNT_CHECK_ENABLE = + FEDERATION_ROUTER_PREFIX + "admin.mount.check.enable"; + public static final boolean DFS_ROUTER_ADMIN_MOUNT_CHECK_ENABLE_DEFAULT = + false; // HDFS Router-based federation web public static final String DFS_ROUTER_HTTP_ENABLE = @@ -308,9 +312,9 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic { public static final String DFS_ROUTER_QUOTA_ENABLE = FEDERATION_ROUTER_PREFIX + "quota.enable"; public static final boolean DFS_ROUTER_QUOTA_ENABLED_DEFAULT = false; - public static final String DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL = + public static final String DFS_ROUTER_QUOTA_CACHE_UPDATE_INTERVAL = FEDERATION_ROUTER_PREFIX + "quota-cache.update.interval"; - public static final long DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL_DEFAULT = + public static final long DFS_ROUTER_QUOTA_CACHE_UPDATE_INTERVAL_DEFAULT = 60000; // HDFS Router security diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java index 64fdabe43b18c..d6e5a1cfe9626 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java @@ -56,7 +56,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Router that provides a unified view of multiple federated HDFS clusters. It diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java index 50094b02d47be..b44142fb17055 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java @@ -25,11 +25,12 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -89,10 +90,11 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.protobuf.BlockingService; /** @@ -126,6 +128,7 @@ public class RouterAdminServer extends AbstractService private static boolean isPermissionEnabled; private boolean iStateStoreCache; private final long maxComponentLength; + private boolean mountTableCheckDestination; public RouterAdminServer(Configuration conf, Router router) throws IOException { @@ -184,6 +187,9 @@ public RouterAdminServer(Configuration conf, Router router) this.maxComponentLength = (int) conf.getLongBytes( RBFConfigKeys.DFS_ROUTER_ADMIN_MAX_COMPONENT_LENGTH_KEY, RBFConfigKeys.DFS_ROUTER_ADMIN_MAX_COMPONENT_LENGTH_DEFAULT); + this.mountTableCheckDestination = conf.getBoolean( + RBFConfigKeys.DFS_ROUTER_ADMIN_MOUNT_CHECK_ENABLE, + RBFConfigKeys.DFS_ROUTER_ADMIN_MOUNT_CHECK_ENABLE_DEFAULT); GenericRefreshProtocolServerSideTranslatorPB genericRefreshXlator = new GenericRefreshProtocolServerSideTranslatorPB(this); @@ -326,6 +332,13 @@ public AddMountTableEntryResponse addMountTableEntry( // Checks max component length limit. MountTable mountTable = request.getEntry(); verifyMaxComponentLength(mountTable); + if (this.mountTableCheckDestination) { + List nsIds = verifyFileInDestinations(mountTable); + if (!nsIds.isEmpty()) { + throw new IllegalArgumentException("File not found in downstream " + + "nameservices: " + StringUtils.join(",", nsIds)); + } + } return getMountTableStore().addMountTableEntry(request); } @@ -336,6 +349,13 @@ public UpdateMountTableEntryResponse updateMountTableEntry( MountTable oldEntry = null; // Checks max component length limit. verifyMaxComponentLength(updateEntry); + if (this.mountTableCheckDestination) { + List nsIds = verifyFileInDestinations(updateEntry); + if (!nsIds.isEmpty()) { + throw new IllegalArgumentException("File not found in downstream " + + "nameservices: " + StringUtils.join(",", nsIds)); + } + } if (this.router.getSubclusterResolver() instanceof MountTableResolver) { MountTableResolver mResolver = (MountTableResolver) this.router.getSubclusterResolver(); @@ -542,10 +562,31 @@ public RefreshMountTableEntriesResponse refreshMountTableEntries( @Override public GetDestinationResponse getDestination( GetDestinationRequest request) throws IOException { + RouterRpcServer rpcServer = this.router.getRpcServer(); + List locations = + rpcServer.getLocationsForPath(request.getSrcPath(), false); + List nsIds = getDestinationNameServices(request, locations); + if (nsIds.isEmpty() && !locations.isEmpty()) { + String nsId = locations.get(0).getNameserviceId(); + nsIds.add(nsId); + } + return GetDestinationResponse.newInstance(nsIds); + } + + /** + * Get destination nameservices where the file in request exists. + * + * @param request request with src info. + * @param locations remote locations to check against. + * @return list of nameservices where the dest file was found + * @throws IOException + */ + private List getDestinationNameServices( + GetDestinationRequest request, List locations) + throws IOException { final String src = request.getSrcPath(); final List nsIds = new ArrayList<>(); RouterRpcServer rpcServer = this.router.getRpcServer(); - List locations = rpcServer.getLocationsForPath(src, false); RouterRpcClient rpcClient = rpcServer.getRPCClient(); RemoteMethod method = new RemoteMethod("getFileInfo", new Class[] {String.class}, new RemoteParam()); @@ -562,11 +603,35 @@ public GetDestinationResponse getDestination( LOG.error("Cannot get location for {}: {}", src, ioe.getMessage()); } - if (nsIds.isEmpty() && !locations.isEmpty()) { - String nsId = locations.get(0).getNameserviceId(); - nsIds.add(nsId); + return nsIds; + } + + /** + * Verify the file exists in destination nameservices to avoid dangling + * mount points. + * + * @param entry the new mount points added, could be from add or update. + * @return destination nameservices where the file doesn't exist. + * @throws IOException unable to verify the file in destinations + */ + public List verifyFileInDestinations(MountTable entry) + throws IOException { + GetDestinationRequest request = + GetDestinationRequest.newInstance(entry.getSourcePath()); + List locations = entry.getDestinations(); + List nsId = + getDestinationNameServices(request, locations); + + // get nameservices where no target file exists + Set destNs = new HashSet<>(nsId); + List nsWithoutFile = new ArrayList<>(); + for (RemoteLocation location : locations) { + String ns = location.getNameserviceId(); + if (!destNs.contains(ns)) { + nsWithoutFile.add(ns); + } } - return GetDestinationResponse.newInstance(nsIds); + return nsWithoutFile; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java index 680cdc93250ec..f62f553e036b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java @@ -73,6 +73,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.UnresolvedPathException; import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; @@ -96,7 +97,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.io.FileNotFoundException; import java.io.IOException; @@ -287,7 +288,7 @@ public HdfsFileStatus create(String src, FsPermission masked, rpcServer.getLocationsForPath(src, true); RemoteLocation createLocation = null; try { - createLocation = rpcServer.getCreateLocation(src); + createLocation = rpcServer.getCreateLocation(src, locations); return rpcClient.invokeSingle(createLocation, method, HdfsFileStatus.class); } catch (IOException ioe) { @@ -305,7 +306,7 @@ public HdfsFileStatus create(String src, FsPermission masked, * @return If caused by an unavailable subcluster. False if the should not be * retried (e.g., NSQuotaExceededException). */ - private static boolean isUnavailableSubclusterException( + protected static boolean isUnavailableSubclusterException( final IOException ioe) { if (ioe instanceof ConnectException || ioe instanceof ConnectTimeoutException || @@ -1314,6 +1315,12 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() return snapshotProto.getSnapshottableDirListing(); } + @Override + public SnapshotStatus[] getSnapshotListing(String snapshotRoot) + throws IOException { + return snapshotProto.getSnapshotListing(snapshotRoot); + } + @Override public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot, String earlierSnapshotName, String laterSnapshotName) throws IOException { @@ -1792,10 +1799,11 @@ public void satisfyStoragePolicy(String path) throws IOException { } @Override - public HAServiceProtocol.HAServiceState getHAServiceState() - throws IOException { - rpcServer.checkOperation(NameNode.OperationCategory.READ, false); - return null; + public HAServiceProtocol.HAServiceState getHAServiceState() { + if (rpcServer.isSafeMode()) { + return HAServiceProtocol.HAServiceState.STANDBY; + } + return HAServiceProtocol.HAServiceState.ACTIVE; } /** @@ -1856,6 +1864,8 @@ private RemoteLocation getFirstMatchingLocation(RemoteLocation location, /** * Aggregate content summaries for each subcluster. + * If the mount point has multiple destinations + * add the quota set value only once. * * @param summaries Collection of individual summaries. * @return Aggregated content summary. @@ -1878,9 +1888,9 @@ private ContentSummary aggregateContentSummary( length += summary.getLength(); fileCount += summary.getFileCount(); directoryCount += summary.getDirectoryCount(); - quota += summary.getQuota(); + quota = summary.getQuota(); spaceConsumed += summary.getSpaceConsumed(); - spaceQuota += summary.getSpaceQuota(); + spaceQuota = summary.getSpaceQuota(); // We return from the first response as we assume that the EC policy // of each sub-cluster is same. if (ecPolicy.isEmpty()) { @@ -1974,7 +1984,8 @@ private static FsPermission getParentPermission(final FsPermission mask) { * @param date Map with the dates. * @return New HDFS file status representing a mount point. */ - private HdfsFileStatus getMountPointStatus( + @VisibleForTesting + HdfsFileStatus getMountPointStatus( String name, int childrenNum, long date) { long modTime = date; long accessTime = date; @@ -2025,6 +2036,8 @@ private HdfsFileStatus getMountPointStatus( } } long inodeId = 0; + Path path = new Path(name); + String nameStr = path.getName(); return new HdfsFileStatus.Builder() .isdir(true) .mtime(modTime) @@ -2033,7 +2046,7 @@ private HdfsFileStatus getMountPointStatus( .owner(owner) .group(group) .symlink(new byte[0]) - .path(DFSUtil.string2Bytes(name)) + .path(DFSUtil.string2Bytes(nameStr)) .fileId(inodeId) .children(childrenNum) .flags(flags) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java index c497d85335922..37407c2a3b8aa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java @@ -21,7 +21,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.federation.store.CachedRecordStore; import org.apache.hadoop.hdfs.server.federation.store.MembershipStore; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java index 5ff03d1a1aa98..a4a7d9e9ddc2d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java @@ -70,8 +70,8 @@ public RouterQuotaUpdateService(final Router router) throws IOException { @Override protected void serviceInit(Configuration conf) throws Exception { this.setIntervalMs(conf.getTimeDuration( - RBFConfigKeys.DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL, - RBFConfigKeys.DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL_DEFAULT, + RBFConfigKeys.DFS_ROUTER_QUOTA_CACHE_UPDATE_INTERVAL, + RBFConfigKeys.DFS_ROUTER_QUOTA_CACHE_UPDATE_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS)); super.serviceInit(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java index dae4b9356436c..8ae3731d6a854 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hdfs.server.federation.router; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY; @@ -29,6 +31,7 @@ import java.lang.reflect.Method; import java.net.ConnectException; import java.net.InetSocketAddress; +import java.net.SocketException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -65,8 +68,11 @@ import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryPolicy.RetryAction.RetryDecision; +import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.RetriableException; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.ipc.Server.Call; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.net.ConnectTimeoutException; import org.apache.hadoop.security.UserGroupInformation; @@ -74,8 +80,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; /** * A client proxy for Router to NN communication using the NN ClientProtocol. @@ -114,11 +120,14 @@ public class RouterRpcClient { private final RetryPolicy retryPolicy; /** Optional perf monitor. */ private final RouterRpcMonitor rpcMonitor; + /** Field separator of CallerContext. */ + private final String contextFieldSeparator; /** Pattern to parse a stack trace line. */ private static final Pattern STACK_TRACE_PATTERN = Pattern.compile("\\tat (.*)\\.(.*)\\((.*):(\\d*)\\)"); + private static final String CLIENT_IP_STR = "clientIp"; /** * Create a router RPC client to manage remote procedure calls to NNs. @@ -135,6 +144,9 @@ public RouterRpcClient(Configuration conf, Router router, this.namenodeResolver = resolver; Configuration clientConf = getClientConfiguration(conf); + this.contextFieldSeparator = + clientConf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, + HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT); this.connectionManager = new ConnectionManager(clientConf); this.connectionManager.start(); @@ -404,6 +416,8 @@ private Object invokeMethod( + router.getRouterId()); } + appendClientIpToCallerContextIfAbsent(); + Object ret = null; if (rpcMonitor != null) { rpcMonitor.proxyOp(); @@ -518,6 +532,29 @@ private Object invokeMethod( } } + /** + * For tracking which is the actual client address. + * It adds trace info "clientIp:ip" to caller context if it's absent. + */ + private void appendClientIpToCallerContextIfAbsent() { + String clientIpInfo = CLIENT_IP_STR + ":" + Server.getRemoteAddress(); + final CallerContext ctx = CallerContext.getCurrent(); + if (isClientIpInfoAbsent(clientIpInfo, ctx)) { + String origContext = ctx == null ? null : ctx.getContext(); + byte[] origSignature = ctx == null ? null : ctx.getSignature(); + CallerContext.setCurrent( + new CallerContext.Builder(origContext, contextFieldSeparator) + .append(clientIpInfo) + .setSignature(origSignature) + .build()); + } + } + + private boolean isClientIpInfoAbsent(String clientIpInfo, CallerContext ctx){ + return ctx == null || ctx.getContext() == null + || !ctx.getContext().contains(clientIpInfo); + } + /** * Invokes a method on the designated object. Catches exceptions specific to * the invocation. @@ -582,9 +619,9 @@ private Object invoke(String nsId, int retryCount, final Method method, * @return If the exception comes from an unavailable subcluster. */ public static boolean isUnavailableException(IOException ioe) { - if (ioe instanceof ConnectException || - ioe instanceof ConnectTimeoutException || + if (ioe instanceof ConnectTimeoutException || ioe instanceof EOFException || + ioe instanceof SocketException || ioe instanceof StandbyException) { return true; } @@ -849,6 +886,45 @@ public T invokeSequential( final List locations, final RemoteMethod remoteMethod, Class expectedResultClass, Object expectedResultValue) throws IOException { + return (T) invokeSequential(remoteMethod, locations, expectedResultClass, + expectedResultValue).getResult(); + } + + /** + * Invokes sequential proxy calls to different locations. Continues to invoke + * calls until the success condition is met, or until all locations have been + * attempted. + * + * The success condition may be specified by: + *
    + *
  • An expected result class + *
  • An expected result value + *
+ * + * If no expected result class/values are specified, the success condition is + * a call that does not throw a remote exception. + * + * This returns RemoteResult, which contains the invoked location as well + * as the result. + * + * @param The type of the remote location. + * @param The type of the remote method return. + * @param remoteMethod The remote method and parameters to invoke. + * @param locations List of locations/nameservices to call concurrently. + * @param expectedResultClass In order to be considered a positive result, the + * return type must be of this class. + * @param expectedResultValue In order to be considered a positive result, the + * return value must equal the value of this object. + * @return The result of the first successful call, or if no calls are + * successful, the result of the first RPC call executed, along with + * the invoked location in form of RemoteResult. + * @throws IOException if the success condition is not met, return the first + * remote exception generated. + */ + public RemoteResult invokeSequential( + final RemoteMethod remoteMethod, final List locations, + Class expectedResultClass, Object expectedResultValue) + throws IOException { final UserGroupInformation ugi = RouterRpcServer.getRemoteUser(); final Method m = remoteMethod.getMethod(); @@ -867,9 +943,9 @@ public T invokeSequential( if (isExpectedClass(expectedResultClass, result) && isExpectedValue(expectedResultValue, result)) { // Valid result, stop here - @SuppressWarnings("unchecked") - T ret = (T)result; - return ret; + @SuppressWarnings("unchecked") R location = (R) loc; + @SuppressWarnings("unchecked") T ret = (T) result; + return new RemoteResult<>(location, ret); } if (firstResult == null) { firstResult = result; @@ -907,9 +983,8 @@ public T invokeSequential( throw thrownExceptions.get(0); } // Return the first result, whether it is the value or not - @SuppressWarnings("unchecked") - T ret = (T)firstResult; - return ret; + @SuppressWarnings("unchecked") T ret = (T) firstResult; + return new RemoteResult<>(locations.get(0), ret); } /** @@ -1239,6 +1314,9 @@ public Map invokeConcurrent( List orderedLocations = new ArrayList<>(); List> callables = new ArrayList<>(); + // transfer originCall & callerContext to worker threads of executor. + final Call originCall = Server.getCurCall().get(); + final CallerContext originContext = CallerContext.getCurrent(); for (final T location : locations) { String nsId = location.getNameserviceId(); final List namenodes = @@ -1256,12 +1334,20 @@ public Map invokeConcurrent( nnLocation = (T)new RemoteLocation(nsId, nnId, location.getDest()); } orderedLocations.add(nnLocation); - callables.add(() -> invokeMethod(ugi, nnList, proto, m, paramList)); + callables.add( + () -> { + transferThreadLocalContext(originCall, originContext); + return invokeMethod(ugi, nnList, proto, m, paramList); + }); } } else { // Call the objectGetter in order of nameservices in the NS list orderedLocations.add(location); - callables.add(() -> invokeMethod(ugi, namenodes, proto, m, paramList)); + callables.add( + () -> { + transferThreadLocalContext(originCall, originContext); + return invokeMethod(ugi, namenodes, proto, m, paramList); + }); } } @@ -1328,6 +1414,20 @@ public Map invokeConcurrent( } } + /** + * Transfer origin thread local context which is necessary to current + * worker thread when invoking method concurrently by executor service. + * + * @param originCall origin Call required for getting remote client ip. + * @param originContext origin CallerContext which should be transferred + * to server side. + */ + private void transferThreadLocalContext( + final Call originCall, final CallerContext originContext) { + Server.getCurCall().set(originCall); + CallerContext.setCurrent(originContext); + } + /** * Get a prioritized list of NNs that share the same nameservice ID (in the * same namespace). NNs that are reported as ACTIVE will be first in the list. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index 5905a1dbbd370..315f864c75a70 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -37,6 +37,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -49,13 +50,13 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; +import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.CryptoProtocolVersion; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries; @@ -112,6 +113,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.NamenodeProtocolService; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ClientNamenodeProtocol; @@ -170,7 +172,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.protobuf.BlockingService; /** @@ -560,8 +562,7 @@ void checkOperation(OperationCategory op) * client requests. */ private void checkSafeMode() throws StandbyException { - RouterSafemodeService safemodeService = router.getSafemodeService(); - if (safemodeService != null && safemodeService.isInSafeMode()) { + if (isSafeMode()) { // Throw standby exception, router is not available if (rpcMonitor != null) { rpcMonitor.routerFailureSafemode(); @@ -572,6 +573,16 @@ private void checkSafeMode() throws StandbyException { } } + /** + * Return true if the Router is in safe mode. + * + * @return true if the Router is in safe mode. + */ + boolean isSafeMode() { + RouterSafemodeService safemodeService = router.getSafemodeService(); + return (safemodeService != null && safemodeService.isInSafeMode()); + } + /** * Get the name of the method that is calling this function. * @@ -586,6 +597,7 @@ static String getMethodName() { /** * Invokes the method at default namespace, if default namespace is not * available then at the first available namespace. + * If the namespace is unavailable, retry once with other namespace. * @param expected return type. * @param method the remote method. * @return the response received after invoking method. @@ -594,18 +606,61 @@ static String getMethodName() { T invokeAtAvailableNs(RemoteMethod method, Class clazz) throws IOException { String nsId = subclusterResolver.getDefaultNamespace(); - if (!nsId.isEmpty()) { - return rpcClient.invokeSingle(nsId, method, clazz); - } // If default Ns is not present return result from first namespace. Set nss = namenodeResolver.getNamespaces(); + try { + if (!nsId.isEmpty()) { + return rpcClient.invokeSingle(nsId, method, clazz); + } + // If no namespace is available, throw IOException. + IOException io = new IOException("No namespace available."); + return invokeOnNs(method, clazz, io, nss); + } catch (IOException ioe) { + if (!clientProto.isUnavailableSubclusterException(ioe)) { + LOG.debug("{} exception cannot be retried", + ioe.getClass().getSimpleName()); + throw ioe; + } + Set nssWithoutFailed = getNameSpaceInfo(nsId); + return invokeOnNs(method, clazz, ioe, nssWithoutFailed); + } + } + + /** + * Invoke the method on first available namespace, + * throw no namespace available exception, if no namespaces are available. + * @param method the remote method. + * @param clazz Class for the return type. + * @param ioe IOException . + * @param nss List of name spaces in the federation + * @return the response received after invoking method. + * @throws IOException + */ + T invokeOnNs(RemoteMethod method, Class clazz, IOException ioe, + Set nss) throws IOException { if (nss.isEmpty()) { - throw new IOException("No namespace available."); + throw ioe; } - nsId = nss.iterator().next().getNameserviceId(); + String nsId = nss.iterator().next().getNameserviceId(); return rpcClient.invokeSingle(nsId, method, clazz); } + /** + * Get set of namespace info's removing the already invoked namespaceinfo. + * @param nsId already invoked namespace id + * @return List of name spaces in the federation on + * removing the already invoked namespaceinfo. + */ + private Set getNameSpaceInfo(String nsId) { + Set namespaceInfos = new HashSet<>(); + for (FederationNamespaceInfo ns : namespaceInfos) { + if (!nsId.equals(ns.getNameserviceId())) { + namespaceInfos.add(ns); + } + } + return namespaceInfos; + } + @Override // ClientProtocol public Token getDelegationToken(Text renewer) throws IOException { @@ -1130,6 +1185,12 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() return clientProto.getSnapshottableDirListing(); } + @Override // ClientProtocol + public SnapshotStatus[] getSnapshotListing(String snapshotRoot) + throws IOException { + return clientProto.getSnapshotListing(snapshotRoot); + } + @Override // ClientProtocol public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot, String earlierSnapshotName, String laterSnapshotName) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSnapshot.java index 7b08092d6431a..9a47f2a012a0a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSnapshot.java @@ -24,10 +24,12 @@ import java.util.Map.Entry; import java.util.Set; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo; import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; @@ -102,10 +104,12 @@ public String createSnapshot(String snapshotRoot, String snapshotName) result = firstelement.getValue(); result = result.replaceFirst(loc.getDest(), loc.getSrc()); } else { - result = rpcClient.invokeSequential( - locations, method, String.class, null); - RemoteLocation loc = locations.get(0); - result = result.replaceFirst(loc.getDest(), loc.getSrc()); + RemoteResult response = + rpcClient.invokeSequential(method, locations, String.class, null); + RemoteLocation loc = response.getLocation(); + String invokedResult = response.getResult(); + result = invokedResult + .replaceFirst(loc.getDest(), loc.getSrc()); } return result; } @@ -157,6 +161,41 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() return RouterRpcServer.merge(ret, SnapshottableDirectoryStatus.class); } + public SnapshotStatus[] getSnapshotListing(String snapshotRoot) + throws IOException { + rpcServer.checkOperation(NameNode.OperationCategory.READ); + final List locations = + rpcServer.getLocationsForPath(snapshotRoot, true, false); + RemoteMethod remoteMethod = new RemoteMethod("getSnapshotListing", + new Class[]{String.class}, + new RemoteParam()); + SnapshotStatus[] response; + if (rpcServer.isInvokeConcurrent(snapshotRoot)) { + Map ret = rpcClient.invokeConcurrent( + locations, remoteMethod, true, false, SnapshotStatus[].class); + response = ret.values().iterator().next(); + String src = ret.keySet().iterator().next().getSrc(); + String dst = ret.keySet().iterator().next().getDest(); + for (SnapshotStatus s : response) { + String mountPath = DFSUtil.bytes2String(s.getParentFullPath()). + replaceFirst(src, dst); + s.setParentFullPath(DFSUtil.string2Bytes(mountPath)); + } + } else { + RemoteResult invokedResponse = rpcClient + .invokeSequential(remoteMethod, locations, SnapshotStatus[].class, + null); + RemoteLocation loc = invokedResponse.getLocation(); + response = invokedResponse.getResult(); + for (SnapshotStatus s : response) { + String mountPath = DFSUtil.bytes2String(s.getParentFullPath()). + replaceFirst(loc.getDest(), loc.getSrc()); + s.setParentFullPath(DFSUtil.string2Bytes(mountPath)); + } + } + return response; + } + public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot, String earlierSnapshotName, String laterSnapshotName) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/RouterSecurityManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/RouterSecurityManager.java index 8e7a34381cff7..7b0787f0f1613 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/RouterSecurityManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/RouterSecurityManager.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.federation.router.security; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/package-info.java index a51e4552955a5..ae65c8fe6755d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/package-info.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/package-info.java @@ -18,7 +18,8 @@ /** * Includes implementations of token secret managers. - * Implementations should extend {@link AbstractDelegationTokenSecretManager}. + * Implementations should extend + * {@link org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager}. */ @InterfaceAudience.Private @InterfaceStability.Evolving diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java index 66c288238eb76..95a38588324cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java @@ -53,7 +53,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * A service to initialize a diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java index 15fc9c1ae9f6c..8352bca12e9aa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java @@ -44,7 +44,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * {@link StateStoreDriver} implementation based on files. In this approach, we diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java index 60dbcdc10ef44..cedc784e39f15 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java @@ -36,7 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.io.Files; +import org.apache.hadoop.thirdparty.com.google.common.io.Files; /** * StateStoreDriver implementation based on a local file. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java index 86721eaa476b6..6b39e20bd7ee7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java @@ -21,7 +21,7 @@ import org.apache.hadoop.util.Time; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Abstract base of a data record in the StateStore. All StateStore records are diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java index 282fe6cbb53e2..907a4055adb82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java @@ -430,6 +430,8 @@ public int hashCode() { .append(this.isReadOnly()) .append(this.getDestOrder()) .append(this.isFaultTolerant()) + .append(this.getQuota().getQuota()) + .append(this.getQuota().getSpaceQuota()) .toHashCode(); } @@ -443,6 +445,9 @@ public boolean equals(Object obj) { .append(this.isReadOnly(), other.isReadOnly()) .append(this.getDestOrder(), other.getDestOrder()) .append(this.isFaultTolerant(), other.isFaultTolerant()) + .append(this.getQuota().getQuota(), other.getQuota().getQuota()) + .append(this.getQuota().getSpaceQuota(), + other.getQuota().getSpaceQuota()) .isEquals(); } return false; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/ConsistentHashRing.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/ConsistentHashRing.java index fc3e49ff9dbb0..ab7bfb16cb822 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/ConsistentHashRing.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/ConsistentHashRing.java @@ -33,8 +33,8 @@ * or remove nodes, it minimizes the item migration. */ public class ConsistentHashRing { - private static final String SEPERATOR = "/"; - private static final String VIRTUAL_NODE_FORMAT = "%s" + SEPERATOR + "%d"; + private static final String SEPARATOR = "/"; + private static final String VIRTUAL_NODE_FORMAT = "%s" + SEPARATOR + "%d"; /** Hash ring. */ private SortedMap ring = new TreeMap(); @@ -119,7 +119,7 @@ public String getLocation(String item) { hash = tailMap.isEmpty() ? ring.firstKey() : tailMap.firstKey(); } String virtualNode = ring.get(hash); - int index = virtualNode.lastIndexOf(SEPERATOR); + int index = virtualNode.lastIndexOf(SEPARATOR); if (index >= 0) { return virtualNode.substring(0, index); } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml index 4bd2ac36b1377..aab90e460654f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml @@ -215,6 +215,16 @@ + + dfs.federation.router.admin.mount.check.enable + false + + If true, add/update mount table will include a destination check to make + sure the file exists in downstream namenodes, and changes to mount table + will fail if the file doesn't exist in any of the destination namenode. + + + dfs.federation.router.http-address 0.0.0.0:50071 diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html index 1f45f4d16c1f7..80b38e7165f78 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html @@ -310,7 +310,7 @@

- + diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md index de45645db093c..66f039a0c8881 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md @@ -201,7 +201,8 @@ And to stop it: ### Mount table management -The mount table entries are pretty much the same as in [ViewFs](../hadoop-hdfs/ViewFs.html). +The mount table entries are pretty much the same as in [ViewFs](../hadoop-hdfs/ViewFs.html). Please make sure the downstream namespace path +exists before creating mount table entry pointing to it. A good practice for simplifying the management is to name the federated namespace with the same names as the destination namespaces. For example, if we to mount `/data/app1` in the federated namespace, it is recommended to have that same name as in the destination namespace. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractRootDirectory.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractRootDirectory.java index dd2bbff7d8da2..b1e4a05500fda 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractRootDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractRootDirectory.java @@ -71,4 +71,9 @@ public void testRmRootRecursive() { public void testRmEmptyRootDirRecursive() { // It doesn't apply because we still have the mount points here } + + @Override + public void testSimpleRootListing() { + // It doesn't apply because DFSRouter dosn't support LISTSTATUS_BATCH. + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMultipleDestinationResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMultipleDestinationResolver.java index 3915c564bf4d6..c4b211c6ee87a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMultipleDestinationResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMultipleDestinationResolver.java @@ -360,6 +360,15 @@ public void testRandomResolver() throws IOException { assertEquals(3, destinations.size()); } + @Test + public void testPrioritizeDestination() throws IOException { + PathLocation dest0 = resolver.getDestinationForPath("/hashall/file0.txt"); + assertDest("subcluster0", dest0); + PathLocation prioritizedDest = PathLocation.prioritizeDestination(dest0, + "subcluster1"); + assertDest("subcluster1", prioritizedDest); + } + /** * Test that a path has files distributed across destinations evenly. * @param path Path to check. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java index 97cf94dc85f62..44c0fc7ed3095 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java @@ -274,9 +274,18 @@ public void testNamenodeHeartBeatEnableDefault() throws IOException { */ private void checkNamenodeHeartBeatEnableDefault(boolean enable) throws IOException { - final Router router = new Router(); - try { + try (Router router = new Router()) { + // Use default config Configuration config = new HdfsConfiguration(); + // bind to any available port + config.set(RBFConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY, "0.0.0.0"); + config.set(RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY, "127.0.0.1:0"); + config.set(RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY, "127.0.0.1:0"); + config.set(RBFConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY, "0.0.0.0"); + config.set(RBFConfigKeys.DFS_ROUTER_HTTP_ADDRESS_KEY, "127.0.0.1:0"); + config.set(RBFConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_KEY, "127.0.0.1:0"); + config.set(RBFConfigKeys.DFS_ROUTER_HTTP_BIND_HOST_KEY, "0.0.0.0"); + config.setBoolean(RBFConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE, enable); router.init(config); if (enable) { @@ -284,8 +293,6 @@ private void checkNamenodeHeartBeatEnableDefault(boolean enable) } else { assertNull(router.getNamenodeHeartbeatServices()); } - } finally { - router.close(); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java index 8a57224b45599..abeeb73a81cd2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java @@ -27,11 +27,14 @@ import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; @@ -57,7 +60,6 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.test.Whitebox; import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.util.Time; import org.junit.AfterClass; @@ -65,6 +67,9 @@ import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; +import org.mockito.internal.util.reflection.FieldSetter; + +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * The administrator interface of the {@link Router} implemented by @@ -78,6 +83,7 @@ public class TestRouterAdmin { "Hadoop:service=Router,name=FederationRPC"; private static List mockMountTable; private static StateStoreService stateStore; + private static RouterRpcClient mockRpcClient; @BeforeClass public static void globalSetUp() throws Exception { @@ -88,6 +94,7 @@ public static void globalSetUp() throws Exception { .admin() .rpc() .build(); + conf.setBoolean(RBFConfigKeys.DFS_ROUTER_ADMIN_MOUNT_CHECK_ENABLE, true); cluster.addRouterOverrides(conf); cluster.startRouters(); routerContext = cluster.getRandomRouter(); @@ -103,11 +110,51 @@ public static void globalSetUp() throws Exception { createNamenodeReport("ns1", "nn1", HAServiceState.ACTIVE)); stateStore.refreshCaches(true); + setUpMocks(); + } + + /** + * Group all mocks together. + * + * @throws IOException + * @throws NoSuchFieldException + */ + private static void setUpMocks() throws IOException, NoSuchFieldException { RouterRpcServer spyRpcServer = Mockito.spy(routerContext.getRouter().createRpcServer()); - Whitebox - .setInternalState(routerContext.getRouter(), "rpcServer", spyRpcServer); + FieldSetter.setField(routerContext.getRouter(), + Router.class.getDeclaredField("rpcServer"), spyRpcServer); Mockito.doReturn(null).when(spyRpcServer).getFileInfo(Mockito.anyString()); + + // mock rpc client for destination check when editing mount tables. + mockRpcClient = Mockito.spy(spyRpcServer.getRPCClient()); + FieldSetter.setField(spyRpcServer, + RouterRpcServer.class.getDeclaredField("rpcClient"), + mockRpcClient); + RemoteLocation remoteLocation0 = + new RemoteLocation("ns0", "/testdir", null); + RemoteLocation remoteLocation1 = + new RemoteLocation("ns1", "/", null); + final Map mockResponse0 = new HashMap<>(); + final Map mockResponse1 = new HashMap<>(); + mockResponse0.put(remoteLocation0, + new HdfsFileStatus.Builder().build()); + Mockito.doReturn(mockResponse0).when(mockRpcClient).invokeConcurrent( + Mockito.eq(Lists.newArrayList(remoteLocation0)), + Mockito.any(RemoteMethod.class), + Mockito.eq(false), + Mockito.eq(false), + Mockito.eq(HdfsFileStatus.class) + ); + mockResponse1.put(remoteLocation1, + new HdfsFileStatus.Builder().build()); + Mockito.doReturn(mockResponse1).when(mockRpcClient).invokeConcurrent( + Mockito.eq(Lists.newArrayList(remoteLocation1)), + Mockito.any(RemoteMethod.class), + Mockito.eq(false), + Mockito.eq(false), + Mockito.eq(HdfsFileStatus.class) + ); } @AfterClass @@ -332,6 +379,26 @@ public void testGetSingleMountTableEntry() throws IOException { assertEquals(entry.getSourcePath(), "/ns0"); } + @Test + public void testVerifyFileInDestinations() throws IOException { + // this entry has been created in the mock setup + MountTable newEntry = MountTable.newInstance( + "/testpath", Collections.singletonMap("ns0", "/testdir"), + Time.now(), Time.now()); + RouterAdminServer adminServer = + this.routerContext.getRouter().getAdminServer(); + List result = adminServer.verifyFileInDestinations(newEntry); + assertEquals(0, result.size()); + + // this entry was not created in the mock + newEntry = MountTable.newInstance( + "/testpath", Collections.singletonMap("ns0", "/testdir1"), + Time.now(), Time.now()); + result = adminServer.verifyFileInDestinations(newEntry); + assertEquals(1, result.size()); + assertEquals("ns0", result.get(0)); + } + /** * Gets an existing mount table record in the state store. * diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java index 35d729cb3ce24..1daff053ed5a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java @@ -1113,6 +1113,10 @@ public void testSafeModeStatus() throws Exception { // ensure the Router become RUNNING state waitState(RouterServiceState.RUNNING); assertFalse(routerContext.getRouter().getSafemodeService().isInSafeMode()); + final RouterClientProtocol clientProtocol = + routerContext.getRouter().getRpcServer().getClientProtocolModule(); + assertEquals(HAServiceState.ACTIVE, clientProtocol.getHAServiceState()); + assertEquals(0, ToolRunner.run(admin, new String[] {"-safemode", "enter" })); @@ -1125,6 +1129,7 @@ public void testSafeModeStatus() throws Exception { // verify state using RBFMetrics assertEquals(RouterServiceState.SAFEMODE.toString(), jsonString); assertTrue(routerContext.getRouter().getSafemodeService().isInSafeMode()); + assertEquals(HAServiceState.STANDBY, clientProtocol.getHAServiceState()); System.setOut(new PrintStream(out)); assertEquals(0, @@ -1136,6 +1141,7 @@ public void testSafeModeStatus() throws Exception { // verify state assertEquals(RouterServiceState.RUNNING.toString(), jsonString); assertFalse(routerContext.getRouter().getSafemodeService().isInSafeMode()); + assertEquals(HAServiceState.ACTIVE, clientProtocol.getHAServiceState()); out.reset(); assertEquals(0, ToolRunner.run(admin, new String[] {"-safemode", "get" })); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java index f945ded4725fd..a346c1a241a80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java @@ -342,6 +342,37 @@ public void testGetMountPointStatusWithIOException() } } + /** + * Verify the getMountPointStatus result of passing in different parameters. + */ + @Test + public void testGetMountPointStatus() throws IOException { + MountTable addEntry = MountTable.newInstance("/testA/testB/testC/testD", + Collections.singletonMap("ns0", "/testA/testB/testC/testD")); + assertTrue(addMountTable(addEntry)); + RouterClientProtocol clientProtocol = new RouterClientProtocol( + nnFs0.getConf(), routerContext.getRouter().getRpcServer()); + String src = "/"; + String child = "testA"; + Path childPath = new Path(src, child); + HdfsFileStatus dirStatus = + clientProtocol.getMountPointStatus(childPath.toString(), 0, 0); + assertEquals(child, dirStatus.getLocalName()); + + String src1 = "/testA"; + String child1 = "testB"; + Path childPath1 = new Path(src1, child1); + HdfsFileStatus dirStatus1 = + clientProtocol.getMountPointStatus(childPath1.toString(), 0, 0); + assertEquals(child1, dirStatus1.getLocalName()); + + String src2 = "/testA/testB"; + String child2 = "testC"; + Path childPath2 = new Path(src2, child2); + HdfsFileStatus dirStatus2 = + clientProtocol.getMountPointStatus(childPath2.toString(), 0, 0); + assertEquals(child2, dirStatus2.getLocalName()); + } /** * GetListing of testPath through router. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterPolicyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterPolicyProvider.java index 36a00e507633e..c2577e67a06fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterPolicyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterPolicyProvider.java @@ -41,7 +41,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; /** * Test suite covering RouterPolicyProvider. We expect that it contains a diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java index 5326d48be2d31..551ae8a8e0612 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java @@ -99,7 +99,7 @@ public void setUp() throws Exception { .quota() .rpc() .build(); - routerConf.set(RBFConfigKeys.DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL, "2s"); + routerConf.set(RBFConfigKeys.DFS_ROUTER_QUOTA_CACHE_UPDATE_INTERVAL, "2s"); // override some hdfs settings that used in testing space quota Configuration hdfsConf = new Configuration(false); @@ -414,13 +414,13 @@ public void testStorageTypeQuota() throws Exception { QuotaUsage usage = client.getQuotaUsage("/type0"); assertEquals(HdfsConstants.QUOTA_RESET, usage.getQuota()); assertEquals(HdfsConstants.QUOTA_RESET, usage.getSpaceQuota()); - verifyTypeQuotaAndConsume(new long[] {-1, -1, ssQuota * 2, -1, -1}, null, + verifyTypeQuotaAndConsume(new long[] {-1, -1, -1, ssQuota * 2, -1, -1}, null, usage); // Verify /type1 quota on NN1. usage = client.getQuotaUsage("/type1"); assertEquals(HdfsConstants.QUOTA_RESET, usage.getQuota()); assertEquals(HdfsConstants.QUOTA_RESET, usage.getSpaceQuota()); - verifyTypeQuotaAndConsume(new long[] {-1, -1, ssQuota, -1, -1}, null, + verifyTypeQuotaAndConsume(new long[] {-1, -1, -1, ssQuota, -1, -1}, null, usage); FileSystem routerFs = routerContext.getFileSystem(); @@ -431,15 +431,15 @@ public void testStorageTypeQuota() throws Exception { assertEquals(2, u1.getFileAndDirectoryCount()); assertEquals(HdfsConstants.QUOTA_RESET, u1.getSpaceQuota()); assertEquals(fileSize * 3, u1.getSpaceConsumed()); - verifyTypeQuotaAndConsume(new long[] {-1, -1, ssQuota, -1, -1}, - new long[] {0, 0, fileSize * 3, 0, 0}, u1); + verifyTypeQuotaAndConsume(new long[] {-1, -1, -1, ssQuota, -1, -1}, + new long[] {0, 0, 0, fileSize * 3, 0, 0}, u1); // Verify /type0 storage type quota usage on Router. assertEquals(HdfsConstants.QUOTA_RESET, u0.getQuota()); assertEquals(4, u0.getFileAndDirectoryCount()); assertEquals(HdfsConstants.QUOTA_RESET, u0.getSpaceQuota()); assertEquals(fileSize * 3 * 2, u0.getSpaceConsumed()); - verifyTypeQuotaAndConsume(new long[] {-1, -1, ssQuota * 2, -1, -1}, - new long[] {0, 0, fileSize * 3 * 2, 0, 0}, u0); + verifyTypeQuotaAndConsume(new long[] {-1, -1, -1, ssQuota * 2, -1, -1}, + new long[] {0, 0, 0, fileSize * 3 * 2, 0, 0}, u0); } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java index d00b93c43062c..2887c08f15a11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java @@ -25,6 +25,7 @@ import java.io.FileNotFoundException; import java.io.IOException; +import java.net.InetSocketAddress; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -34,8 +35,10 @@ import java.util.TreeSet; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.AclEntry; @@ -43,6 +46,8 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; @@ -56,7 +61,10 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.GetDestinationResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; +import org.apache.hadoop.hdfs.tools.federation.RouterAdmin; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.util.ToolRunner; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -492,6 +500,38 @@ public void testIsMultiDestDir() throws Exception { assertFalse(client.isMultiDestDirectory("/mount/dir")); } + /** + * Verifies the snapshot location returned after snapshot operations is in + * accordance to the mount path. + */ + @Test + public void testSnapshotPathResolution() throws Exception { + // Create a mount entry with non isPathAll order, so as to call + // invokeSequential. + Map destMap = new HashMap<>(); + destMap.put("ns0", "/tmp_ns0"); + destMap.put("ns1", "/tmp_ns1"); + nnFs0.mkdirs(new Path("/tmp_ns0")); + nnFs1.mkdirs(new Path("/tmp_ns1")); + MountTable addEntry = MountTable.newInstance("/mountSnap", destMap); + addEntry.setDestOrder(DestinationOrder.HASH); + assertTrue(addMountTable(addEntry)); + // Create the actual directory in the destination second in sequence of + // invokeSequential. + nnFs0.mkdirs(new Path("/tmp_ns0/snapDir")); + Path snapDir = new Path("/mountSnap/snapDir"); + Path snapshotPath = new Path("/mountSnap/snapDir/.snapshot/snap"); + routerFs.allowSnapshot(snapDir); + // Verify the snapshot path returned after createSnapshot is as per mount + // path. + Path snapshot = routerFs.createSnapshot(snapDir, "snap"); + assertEquals(snapshotPath, snapshot); + // Verify the snapshot path returned as part of snapshotListing is as per + // mount path. + SnapshotStatus[] snapshots = routerFs.getSnapshotListing(snapDir); + assertEquals(snapshotPath, snapshots[0].getFullPath()); + } + @Test public void testRenameMultipleDestDirectories() throws Exception { // Test renaming directories using rename API. @@ -509,6 +549,136 @@ public void testRenameMultipleDestDirectories() throws Exception { verifyRenameOnMultiDestDirectories(DestinationOrder.SPACE, true); } + @Test + public void testClearQuota() throws Exception { + long nsQuota = 5; + long ssQuota = 100; + Path path = new Path("/router_test"); + nnFs0.mkdirs(path); + nnFs1.mkdirs(path); + MountTable addEntry = MountTable.newInstance("/router_test", + Collections.singletonMap("ns0", "/router_test")); + addEntry.setQuota(new RouterQuotaUsage.Builder().build()); + assertTrue(addMountTable(addEntry)); + RouterQuotaUpdateService updateService = + routerContext.getRouter().getQuotaCacheUpdateService(); + updateService.periodicInvoke(); + + //set quota and validate the quota + RouterAdmin admin = getRouterAdmin(); + String[] argv = new String[] {"-setQuota", path.toString(), "-nsQuota", + String.valueOf(nsQuota), "-ssQuota", String.valueOf(ssQuota)}; + assertEquals(0, ToolRunner.run(admin, argv)); + updateService.periodicInvoke(); + resolver.loadCache(true); + ContentSummary cs = routerFs.getContentSummary(path); + assertEquals(nsQuota, cs.getQuota()); + assertEquals(ssQuota, cs.getSpaceQuota()); + + //clear quota and validate the quota + argv = new String[] {"-clrQuota", path.toString()}; + assertEquals(0, ToolRunner.run(admin, argv)); + updateService.periodicInvoke(); + resolver.loadCache(true); + //quota should be cleared + ContentSummary cs1 = routerFs.getContentSummary(path); + assertEquals(-1, cs1.getQuota()); + assertEquals(-1, cs1.getSpaceQuota()); + } + + @Test + public void testContentSummaryWithMultipleDest() throws Exception { + MountTable addEntry; + long nsQuota = 5; + long ssQuota = 100; + Path path = new Path("/testContentSummaryWithMultipleDest"); + Map destMap = new HashMap<>(); + destMap.put("ns0", "/testContentSummaryWithMultipleDest"); + destMap.put("ns1", "/testContentSummaryWithMultipleDest"); + nnFs0.mkdirs(path); + nnFs1.mkdirs(path); + addEntry = + MountTable.newInstance("/testContentSummaryWithMultipleDest", destMap); + addEntry.setQuota( + new RouterQuotaUsage.Builder().quota(nsQuota).spaceQuota(ssQuota) + .build()); + assertTrue(addMountTable(addEntry)); + RouterQuotaUpdateService updateService = + routerContext.getRouter().getQuotaCacheUpdateService(); + updateService.periodicInvoke(); + ContentSummary cs = routerFs.getContentSummary(path); + assertEquals(nsQuota, cs.getQuota()); + assertEquals(ssQuota, cs.getSpaceQuota()); + ContentSummary ns0Cs = nnFs0.getContentSummary(path); + assertEquals(nsQuota, ns0Cs.getQuota()); + assertEquals(ssQuota, ns0Cs.getSpaceQuota()); + ContentSummary ns1Cs = nnFs1.getContentSummary(path); + assertEquals(nsQuota, ns1Cs.getQuota()); + assertEquals(ssQuota, ns1Cs.getSpaceQuota()); + } + + @Test + public void testContentSummaryMultipleDestWithMaxValue() + throws Exception { + MountTable addEntry; + long nsQuota = Long.MAX_VALUE - 2; + long ssQuota = Long.MAX_VALUE - 2; + Path path = new Path("/testContentSummaryMultipleDestWithMaxValue"); + Map destMap = new HashMap<>(); + destMap.put("ns0", "/testContentSummaryMultipleDestWithMaxValue"); + destMap.put("ns1", "/testContentSummaryMultipleDestWithMaxValue"); + nnFs0.mkdirs(path); + nnFs1.mkdirs(path); + addEntry = MountTable + .newInstance("/testContentSummaryMultipleDestWithMaxValue", destMap); + addEntry.setQuota( + new RouterQuotaUsage.Builder().quota(nsQuota).spaceQuota(ssQuota) + .build()); + assertTrue(addMountTable(addEntry)); + RouterQuotaUpdateService updateService = + routerContext.getRouter().getQuotaCacheUpdateService(); + updateService.periodicInvoke(); + ContentSummary cs = routerFs.getContentSummary(path); + assertEquals(nsQuota, cs.getQuota()); + assertEquals(ssQuota, cs.getSpaceQuota()); + } + + /** + * Test write on mount point with multiple destinations + * and making a one of the destination's subcluster unavailable. + */ + @Test + public void testWriteWithUnavailableSubCluster() throws IOException { + //create a mount point with multiple destinations + Path path = new Path("/testWriteWithUnavailableSubCluster"); + Map destMap = new HashMap<>(); + destMap.put("ns0", "/testWriteWithUnavailableSubCluster"); + destMap.put("ns1", "/testWriteWithUnavailableSubCluster"); + nnFs0.mkdirs(path); + nnFs1.mkdirs(path); + MountTable addEntry = + MountTable.newInstance("/testWriteWithUnavailableSubCluster", destMap); + addEntry.setQuota(new RouterQuotaUsage.Builder().build()); + addEntry.setDestOrder(DestinationOrder.RANDOM); + addEntry.setFaultTolerant(true); + assertTrue(addMountTable(addEntry)); + + //make one subcluster unavailable and perform write on mount point + MiniDFSCluster dfsCluster = cluster.getCluster(); + dfsCluster.shutdownNameNode(0); + FSDataOutputStream out = null; + Path filePath = new Path(path, "aa"); + try { + out = routerFs.create(filePath); + out.write("hello".getBytes()); + out.hflush(); + assertTrue(routerFs.exists(filePath)); + } finally { + IOUtils.closeStream(out); + dfsCluster.restartNameNode(0); + } + } + /** * Test to verify rename operation on directories in case of multiple * destinations. @@ -690,4 +860,12 @@ private static FileSystem getFileSystem(final String nsId) { return null; } + private RouterAdmin getRouterAdmin() { + Router router = routerContext.getRouter(); + Configuration configuration = routerContext.getConf(); + InetSocketAddress routerSocket = router.getAdminServerAddress(); + configuration.setSocketAddr(RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY, + routerSocket); + return new RouterAdmin(configuration); + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java index b9b1212333eea..faa3111ea7ed3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.federation.router; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.countContents; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile; @@ -84,8 +85,6 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -94,6 +93,9 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; @@ -110,6 +112,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; @@ -118,6 +121,7 @@ import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; +import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.test.GenericTestUtils; @@ -131,7 +135,7 @@ import org.slf4j.LoggerFactory; import java.util.function.Supplier; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; /** * The the RPC interface of the {@link Router} implemented by @@ -194,8 +198,16 @@ public int compare( @BeforeClass public static void globalSetUp() throws Exception { + Configuration namenodeConf = new Configuration(); + namenodeConf.setBoolean(DFSConfigKeys.HADOOP_CALLER_CONTEXT_ENABLED_KEY, + true); + // It's very easy to become overloaded for some specific dn in this small + // cluster, which will cause the EC file block allocation failure. To avoid + // this issue, we disable considerLoad option. + namenodeConf.setBoolean(DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); cluster = new MiniRouterDFSCluster(false, NUM_SUBCLUSTERS); cluster.setNumDatanodesPerNameservice(NUM_DNS); + cluster.addNamenodeOverrides(namenodeConf); cluster.setIndependentDNs(); Configuration conf = new Configuration(); @@ -926,6 +938,16 @@ public void testGetSnapshotListing() throws IOException { SnapshottableDirectoryStatus snapshotDir0 = dirList[0]; assertEquals(snapshotPath, snapshotDir0.getFullPath().toString()); + // check for snapshot listing through the Router + SnapshotStatus[] snapshots = routerProtocol. + getSnapshotListing(snapshotPath); + assertEquals(2, snapshots.length); + assertEquals(SnapshotTestHelper.getSnapshotRoot( + new Path(snapshotPath), snapshot1), + snapshots[0].getFullPath()); + assertEquals(SnapshotTestHelper.getSnapshotRoot( + new Path(snapshotPath), snapshot2), + snapshots[1].getFullPath()); // Check for difference report in two snapshot SnapshotDiffReport diffReport = routerProtocol.getSnapshotDiffReport( snapshotPath, snapshot1, snapshot2); @@ -1882,4 +1904,27 @@ private DFSClient getFileDFSClient(final String path) { } return null; } + + @Test + public void testMkdirsWithCallerContext() throws IOException { + GenericTestUtils.LogCapturer auditlog = + GenericTestUtils.LogCapturer.captureLogs(FSNamesystem.auditLog); + + // Current callerContext is null + assertNull(CallerContext.getCurrent()); + + // Set client context + CallerContext.setCurrent( + new CallerContext.Builder("clientContext").build()); + + // Create a directory via the router + String dirPath = "/test_dir_with_callercontext"; + FsPermission permission = new FsPermission("755"); + routerProtocol.mkdirs(dirPath, permission, false); + + // The audit log should contains "callerContext=clientContext,clientIp:" + assertTrue(auditlog.getOutput() + .contains("callerContext=clientContext,clientIp:")); + assertTrue(verifyFileExists(routerFS, dirPath)); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java index 30a47a45620fc..4f112ba9b7294 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.lang.reflect.Method; +import java.net.InetAddress; import java.net.URISyntaxException; import java.util.Arrays; import java.util.EnumSet; @@ -67,6 +68,7 @@ import org.apache.hadoop.hdfs.server.namenode.ha.HAContext; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.test.GenericTestUtils; @@ -434,4 +436,45 @@ public void testSubclusterDown() throws Exception { setInternalState(ns0, "haContext", nn0haCtx); setInternalState(router0ClientProtocol, "allowPartialList", true); } + + @Test + public void testCallerContextWithMultiDestinations() throws IOException { + GenericTestUtils.LogCapturer auditLog = + GenericTestUtils.LogCapturer.captureLogs(FSNamesystem.auditLog); + + // set client context + CallerContext.setCurrent( + new CallerContext.Builder("clientContext").build()); + // assert the initial caller context as expected + assertEquals("clientContext", CallerContext.getCurrent().getContext()); + + DistributedFileSystem routerFs = + (DistributedFileSystem) getRouterFileSystem(); + // create a directory via the router + Path dirPath = new Path("/test_caller_context_with_multi_destinations"); + routerFs.mkdirs(dirPath); + // invoke concurrently in RouterRpcClient + routerFs.listStatus(dirPath); + // invoke sequentially in RouterRpcClient + routerFs.getFileStatus(dirPath); + + String auditFlag = "src=" + dirPath.toString(); + String clientIpInfo = "clientIp:" + + InetAddress.getLocalHost().getHostAddress(); + for (String line : auditLog.getOutput().split("\n")) { + if (line.contains(auditFlag)) { + // assert origin caller context exist in audit log + assertTrue(line.contains("callerContext=clientContext")); + String callerContext = line.substring( + line.indexOf("callerContext=clientContext")); + // assert client ip info exist in caller context + assertTrue(callerContext.contains(clientIpInfo)); + // assert client ip info appears only once in caller context + assertEquals(callerContext.indexOf(clientIpInfo), + callerContext.lastIndexOf(clientIpInfo)); + } + } + // clear client context + CallerContext.setCurrent(null); + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterUserMappings.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterUserMappings.java index 19d1c436bc827..ba8c4639e4d13 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterUserMappings.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterUserMappings.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.federation.router; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index e1ea5840deda5..a8c154aa4a9ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -68,8 +68,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> test - com.google.guava - guava + org.apache.hadoop.thirdparty + hadoop-shaded-guava compile @@ -416,7 +416,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> src/main/webapps/static/moment.min.js src/main/webapps/static/dust-full-2.0.0.min.js src/main/webapps/static/dust-helpers-1.1.1.min.js - src/main/webapps/static/jquery-3.4.1.min.js + src/main/webapps/static/jquery-3.5.1.min.js src/main/webapps/static/jquery.dataTables.min.js src/main/webapps/static/json-bignum.js src/main/webapps/static/dataTables.bootstrap.css diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 7a8bf8dbe0deb..fa933540735ca 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -54,6 +54,7 @@ function hadoop_usage hadoop_add_subcommand "jmxget" admin "get JMX exported values from NameNode or DataNode." hadoop_add_subcommand "journalnode" daemon "run the DFS journalnode" hadoop_add_subcommand "lsSnapshottableDir" client "list all snapshottable dirs owned by the current user" + hadoop_add_subcommand "lsSnapshot" client "list all snapshots for a snapshottable directory" hadoop_add_subcommand "mover" daemon "run a utility to move block replicas across storage types" hadoop_add_subcommand "namenode" daemon "run the DFS namenode" hadoop_add_subcommand "nfs3" daemon "run an NFS version 3 gateway" @@ -166,6 +167,9 @@ function hdfscmd_case lsSnapshottableDir) HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir ;; + lsSnapshot) + HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshot + ;; mover) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.mover.Mover diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd index 23d6a5aa1c301..21d4de75cc8b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd @@ -59,7 +59,7 @@ if "%1" == "--loglevel" ( ) ) - set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck fsImageValidation balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath crypto dfsrouter dfsrouteradmin debug + set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck fsImageValidation balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir lsSnapshot cacheadmin mover storagepolicies classpath crypto dfsrouter dfsrouteradmin debug for %%i in ( %hdfscommands% ) do ( if %hdfs-command% == %%i set hdfscommand=true ) @@ -167,6 +167,10 @@ goto :eof set CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir goto :eof +:lsSnapshot + set CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshot + goto :eof + :cacheadmin set CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin goto :eof @@ -253,6 +257,8 @@ goto :eof @echo current directory contents with a snapshot @echo lsSnapshottableDir list all snapshottable dirs owned by the current user @echo Use -help to see options + @echo lsSnapshot list all snapshots for a snapshottable dir + @echo Use -help to see options @echo cacheadmin configure the HDFS cache @echo crypto configure HDFS encryption zones @echo mover run a utility to move block replicas across storage types diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 9de33ff60a62e..5097633d3fe36 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -239,6 +239,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys { HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY; public static final boolean DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_DEFAULT = true; + public static final String + DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYSTORAGETYPE_KEY = + "dfs.namenode.redundancy.considerLoadByStorageType"; + public static final boolean + DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYSTORAGETYPE_DEFAULT = false; public static final String DFS_NAMENODE_READ_CONSIDERLOAD_KEY = "dfs.namenode.read.considerLoad"; public static final boolean DFS_NAMENODE_READ_CONSIDERLOAD_DEFAULT = @@ -500,8 +505,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_SNAPSHOT_MAX_LIMIT = "dfs.namenode.snapshot.max.limit"; - public static final int DFS_NAMENODE_SNAPSHOT_MAX_LIMIT_DEFAULT = 65536; + public static final String + DFS_NAMENODE_SNAPSHOT_FILESYSTEM_LIMIT = + "dfs.namenode.snapshot.filesystem.limit"; + // default value is same as snapshot quota set for a snapshottable directory + public static final int + DFS_NAMENODE_SNAPSHOT_FILESYSTEM_LIMIT_DEFAULT = 65536; + public static final String DFS_NAMENODE_SNAPSHOT_SKIPLIST_SKIP_INTERVAL = "dfs.namenode.snapshot.skiplist.interval"; public static final int DFS_NAMENODE_SNAPSHOT_SKIPLIST_SKIP_INTERVAL_DEFAULT = @@ -679,7 +690,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { 600; /** * The maximum number of getBlocks RPCs data movement utilities can make to - * a NameNode per second. Values <= 0 disable throttling. This affects + * a NameNode per second. Values <= 0 disable throttling. This affects * anything that uses a NameNodeConnector, i.e., the Balancer, Mover, * and StoragePolicySatisfier. */ @@ -855,6 +866,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT = 21 * 24; // 3 weeks. public static final String DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND = "dfs.block.scanner.volume.bytes.per.second"; public static final long DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND_DEFAULT = 1048576L; + /** + * The amount of time in milliseconds that the BlockScanner times out waiting + * for the VolumeScanner thread to join during a shutdown call. + */ + public static final String DFS_BLOCK_SCANNER_VOLUME_JOIN_TIMEOUT_MSEC_KEY = + "dfs.block.scanner.volume.join.timeout.ms"; + public static final long DFS_BLOCK_SCANNER_VOLUME_JOIN_TIMEOUT_MSEC_DEFAULT = + TimeUnit.SECONDS.toMillis(5); public static final String DFS_BLOCK_SCANNER_SKIP_RECENT_ACCESSED = "dfs.block.scanner.skip.recent.accessed"; public static final boolean DFS_BLOCK_SCANNER_SKIP_RECENT_ACCESSED_DEFAULT = @@ -977,7 +996,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_IMAGE_TRANSFER_RATE_KEY = "dfs.image.transfer.bandwidthPerSec"; - public static final long DFS_IMAGE_TRANSFER_RATE_DEFAULT = 0; //no throttling + public static final long DFS_IMAGE_TRANSFER_RATE_DEFAULT = 52428800; public static final String DFS_IMAGE_TRANSFER_BOOTSTRAP_STANDBY_RATE_KEY = "dfs.image.transfer-bootstrap-standby.bandwidthPerSec"; @@ -1380,7 +1399,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.datanode.parallel.volumes.load.threads.num"; public static final String DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS_KEY = "dfs.datanode.block.id.layout.upgrade.threads"; - public static final int DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS = 12; + public static final int DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS = 6; public static final String DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_KEY = "dfs.namenode.inotify.max.events.per.rpc"; @@ -1497,6 +1516,26 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final boolean DFS_PROTECTED_SUBDIRECTORIES_ENABLE_DEFAULT = false; + /** + * HDFS-15548 to allow DISK/ARCHIVE configured on the same disk mount. + * The default ratio will be applied if DISK/ARCHIVE are configured + * on same disk mount. + * + * Beware that capacity usage might be larger than 100% if there are already + * data blocks exist and the configured ratio is small, which will + * prevent the volume from taking new blocks until capacity is balanced out. + */ + public static final String DFS_DATANODE_ALLOW_SAME_DISK_TIERING = + "dfs.datanode.same-disk-tiering.enabled"; + public static final boolean DFS_DATANODE_ALLOW_SAME_DISK_TIERING_DEFAULT = + false; + + public static final String + DFS_DATANODE_RESERVE_FOR_ARCHIVE_DEFAULT_PERCENTAGE = + "dfs.datanode.reserve-for-archive.default.percentage"; + public static final double + DFS_DATANODE_RESERVE_FOR_ARCHIVE_DEFAULT_PERCENTAGE_DEFAULT = 0.0; + // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry @Deprecated public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 00f14cd82e42f..efb2a8bcfc6d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -39,7 +39,6 @@ import java.io.ByteArrayInputStream; import java.io.DataInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; import java.net.InetAddress; @@ -104,11 +103,11 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.ToolRunner; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import org.apache.hadoop.thirdparty.protobuf.BlockingService; @InterfaceAudience.Private @@ -1777,7 +1776,6 @@ public static DelegationTokenIdentifier decodeDelegationToken( * was found. * @throws ParentNotDirectoryException * @throws UnresolvedLinkException - * @throws FileNotFoundException */ public static void checkProtectedDescendants( FSDirectory fsd, INodesInPath iip) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java index 43e76c79e2657..53d3b4b2936cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java @@ -56,9 +56,9 @@ import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.security.UserGroupInformation; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.slf4j.LoggerFactory; @InterfaceAudience.Private diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java index dbc5deace2d70..d3ceb21c80298 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.net; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java index 7a9afabc1edb9..72c89f57872b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.net; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.net.InnerNode; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java index 6d2c0ac3c0391..d9baa8ff45f74 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java @@ -33,8 +33,8 @@ import org.apache.hadoop.hdfs.protocol.BlockListAsLongs.BlockReportReplica; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.datanode.Replica; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java index 2c0d26363b3cf..aef009a03a1dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.protocol; -import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull; import java.util.Date; @@ -28,7 +28,7 @@ import org.apache.hadoop.util.IntrusiveCollection; import org.apache.hadoop.util.IntrusiveCollection.Element; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Namenode class that tracks state related to a cached path. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutFlags.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutFlags.java index fc1717f26a5f9..edc184025cefa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutFlags.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutFlags.java @@ -23,10 +23,10 @@ import org.apache.hadoop.classification.InterfaceAudience; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; /** * LayoutFlags represent features which the FSImage and edit logs can either diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java index 676e8276f258e..ef547788f1ac7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java @@ -82,18 +82,20 @@ public String toString() { } public static class Bean { - private final String snapshotID; + private final int snapshotID; private final String snapshotDirectory; private final long modificationTime; + private final String status; - public Bean(String snapshotID, String snapshotDirectory, - long modificationTime) { + public Bean(int snapshotID, String snapshotDirectory, + long modificationTime, boolean isMarkedAsDeleted) { this.snapshotID = snapshotID; this.snapshotDirectory = snapshotDirectory; this.modificationTime = modificationTime; + this.status = isMarkedAsDeleted ? "DELETED" : "ACTIVE"; } - public String getSnapshotID() { + public int getSnapshotID() { return snapshotID; } @@ -104,5 +106,9 @@ public String getSnapshotDirectory() { public long getModificationTime() { return modificationTime; } + + public String getStatus() { + return status; + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java index ae17761c204ab..0e2dc71930615 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java @@ -21,7 +21,7 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY; import static org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil.*; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -59,8 +59,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Charsets; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * Negotiates SASL for DataTransferProtocol on behalf of a server. There are diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index e0afe006a2f9a..5132afaa4b15c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -66,6 +66,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusRequestProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusResponseProto; @@ -161,6 +162,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportListingResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshottableDirListingRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshottableDirListingResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotListingRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotListingResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePoliciesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePoliciesResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePolicyRequestProto; @@ -325,6 +328,9 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements static final GetSnapshottableDirListingResponseProto NULL_GET_SNAPSHOTTABLE_DIR_LISTING_RESPONSE = GetSnapshottableDirListingResponseProto.newBuilder().build(); + static final GetSnapshotListingResponseProto + NULL_GET_SNAPSHOT_LISTING_RESPONSE = + GetSnapshotListingResponseProto.newBuilder().build(); static final SetStoragePolicyResponseProto VOID_SET_STORAGE_POLICY_RESPONSE = SetStoragePolicyResponseProto.newBuilder().build(); static final UnsetStoragePolicyResponseProto @@ -1349,6 +1355,24 @@ public GetSnapshottableDirListingResponseProto getSnapshottableDirListing( } } + @Override + public GetSnapshotListingResponseProto getSnapshotListing( + RpcController controller, GetSnapshotListingRequestProto request) + throws ServiceException { + try { + SnapshotStatus[] result = server + .getSnapshotListing(request.getSnapshotRoot()); + if (result != null) { + return GetSnapshotListingResponseProto.newBuilder(). + setSnapshotList(PBHelperClient.convert(result)).build(); + } else { + return NULL_GET_SNAPSHOT_LISTING_RESPONSE; + } + } catch (IOException e) { + throw new ServiceException(e); + } + } + @Override public GetSnapshotDiffReportResponseProto getSnapshotDiffReport( RpcController controller, GetSnapshotDiffReportRequestProto request) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java index b512d7e21d57d..add19e9e102ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java @@ -69,7 +69,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.protobuf.RpcController; import org.apache.hadoop.thirdparty.protobuf.ServiceException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java index 5ecbcb7bcd590..9244b9fef8571 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java @@ -61,7 +61,7 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.thirdparty.protobuf.RpcController; import org.apache.hadoop.thirdparty.protobuf.ServiceException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java index 5eead67fa7b5d..bf72723071d26 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; -import com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; /** * Interface for a remote log which is only communicated with asynchronously. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java index f024b0e8c267d..684e7dd69fd40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java @@ -34,12 +34,12 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Maps; -import com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; /** * Wrapper around a set of Loggers, taking care of fanning out diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java index ad947f18f13e9..9908160522651 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java @@ -57,16 +57,16 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.util.StopWatch; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.net.InetAddresses; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.common.util.concurrent.UncaughtExceptionHandlers; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.UncaughtExceptionHandlers; /** * Channel to a remote JournalNode using Hadoop IPC. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java index fde6b99817674..6eef8ffd38620 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java @@ -29,7 +29,7 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableQuantiles; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; /** * The metrics for a journal from the writer's perspective. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java index 501a77ec1d738..e2a169aeb3c5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java @@ -24,17 +24,17 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.MoreExecutors; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.util.StopWatch; import org.apache.hadoop.util.Timer; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; import org.apache.hadoop.thirdparty.protobuf.Message; import org.apache.hadoop.thirdparty.protobuf.TextFormat; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumException.java index 446092ebe1cc2..1f60e3d468821 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumException.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumException.java @@ -22,8 +22,8 @@ import org.apache.hadoop.util.StringUtils; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Exception thrown when too many exceptions occur while gathering diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java index a3ad0900c6044..354b250bcc1f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java @@ -58,10 +58,10 @@ import org.apache.hadoop.log.LogThrottlingHelper; import org.apache.hadoop.log.LogThrottlingHelper.LogAction; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.thirdparty.protobuf.TextFormat; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/SegmentRecoveryComparator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/SegmentRecoveryComparator.java index 61b60aaad875d..4b2a518ac0dff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/SegmentRecoveryComparator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/SegmentRecoveryComparator.java @@ -23,9 +23,9 @@ import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto; import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto; -import com.google.common.base.Preconditions; -import com.google.common.collect.ComparisonChain; -import com.google.common.primitives.Booleans; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Booleans; /** * Compares responses to the prepareRecovery RPC. This is responsible for diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java index b7d652395c631..7f82bff3376f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hdfs.server.namenode.NNStorage; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; -import com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; /** * A {@link Storage} implementation for the {@link JournalNode}. @@ -236,6 +236,10 @@ void format(NamespaceInfo nsInfo, boolean force) throws IOException { void analyzeStorage() throws IOException { this.state = sd.analyzeStorage(StartupOption.REGULAR, this); + refreshStorage(); + } + + void refreshStorage() throws IOException { if (state == StorageState.NORMAL) { readProperties(sd); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java index 70ed4c7159fa4..99a087e2da5eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java @@ -71,10 +71,10 @@ import org.apache.hadoop.util.StopWatch; import org.apache.hadoop.util.Time; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Charsets; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hadoop.thirdparty.protobuf.TextFormat; /** @@ -1178,6 +1178,8 @@ public synchronized void doRollback() throws IOException { // directory will be renamed. It will be reopened lazily on next access. IOUtils.cleanupWithLogger(LOG, committedTxnId); storage.getJournalManager().doRollback(); + // HADOOP-17142: refresh properties after rollback performed. + storage.refreshStorage(); } synchronized void discardSegments(long startTxId) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalFaultInjector.java index cefb7b5294406..f55933fa3536b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalFaultInjector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalFaultInjector.java @@ -19,7 +19,7 @@ import java.io.IOException; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java index 841c77b6d9826..4079cd63bed1f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java @@ -17,11 +17,11 @@ */ package org.apache.hadoop.hdfs.qjournal.server; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Strings; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.util.VersionInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java index ef44f21d2cfcb..d13c98f5c0f1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.qjournal.server; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.protobuf.BlockingService; import org.slf4j.Logger; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java index bb141d8437496..b4997eacd0719 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hdfs.qjournal.server; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java index 3cd7fffc587aa..e0b84d75fb212 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.qjournal.server; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockPoolTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockPoolTokenSecretManager.java index e477eee8437f7..d81bc98ff8aea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockPoolTokenSecretManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockPoolTokenSecretManager.java @@ -27,7 +27,7 @@ import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.StorageType; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java index c01ab56ca2053..1d393783565ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.security.token.block; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.IOException; @@ -44,10 +44,10 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Timer; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.HashMultiset; -import com.google.common.collect.Multiset; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultiset; +import org.apache.hadoop.thirdparty.com.google.common.collect.Multiset; /** * BlockTokenSecretManager can be instantiated in 2 modes, master mode diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java index 49986e9d9827a..b061d4bfa6b10 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java @@ -49,8 +49,8 @@ import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.security.token.delegation.DelegationKey; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.thirdparty.protobuf.ByteString; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java index 2810434609acf..bcf535740d152 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java @@ -16,8 +16,8 @@ */ package org.apache.hadoop.hdfs.server.aliasmap; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 753a2703ec32e..33b5fa46d6e6a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.balancer; -import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument; import static org.apache.hadoop.hdfs.protocol.BlockType.CONTIGUOUS; import java.io.IOException; @@ -36,7 +36,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdfs.DFSUtilClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,7 +69,7 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /**

The balancer is a tool that balances disk space usage on an HDFS cluster * when some datanodes become full or when new empty nodes join the cluster. @@ -282,6 +282,9 @@ static int getFailedTimesSinceLastSuccessfulBalance() { */ Balancer(NameNodeConnector theblockpool, BalancerParameters p, Configuration conf) { + // NameNode configuration parameters for balancing + getInt(conf, DFSConfigKeys.DFS_NAMENODE_GETBLOCKS_MAX_QPS_KEY, + DFSConfigKeys.DFS_NAMENODE_GETBLOCKS_MAX_QPS_DEFAULT); final long movedWinWidth = getLong(conf, DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_DEFAULT); @@ -291,10 +294,6 @@ static int getFailedTimesSinceLastSuccessfulBalance() { final int dispatcherThreads = getInt(conf, DFSConfigKeys.DFS_BALANCER_DISPATCHERTHREADS_KEY, DFSConfigKeys.DFS_BALANCER_DISPATCHERTHREADS_DEFAULT); - final int maxConcurrentMovesPerNode = getInt(conf, - DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY, - DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT); - final long getBlocksSize = getLongBytes(conf, DFSConfigKeys.DFS_BALANCER_GETBLOCKS_SIZE_KEY, DFSConfigKeys.DFS_BALANCER_GETBLOCKS_SIZE_DEFAULT); @@ -311,6 +310,13 @@ static int getFailedTimesSinceLastSuccessfulBalance() { DFSConfigKeys.DFS_BALANCER_MAX_ITERATION_TIME_KEY, DFSConfigKeys.DFS_BALANCER_MAX_ITERATION_TIME_DEFAULT); + // DataNode configuration parameters for balancing + final int maxConcurrentMovesPerNode = getInt(conf, + DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY, + DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT); + getLongBytes(conf, DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY, + DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_DEFAULT); + this.nnc = theblockpool; this.dispatcher = new Dispatcher(theblockpool, p.getIncludedNodes(), @@ -603,12 +609,13 @@ static class Result { this.bytesAlreadyMoved = bytesAlreadyMoved; } - void print(int iteration, PrintStream out) { - out.printf("%-24s %10d %19s %18s %17s%n", + void print(int iteration, NameNodeConnector nnc, PrintStream out) { + out.printf("%-24s %10d %19s %18s %17s %s%n", DateFormat.getDateTimeInstance().format(new Date()), iteration, StringUtils.byteDesc(bytesAlreadyMoved), StringUtils.byteDesc(bytesLeftToMove), - StringUtils.byteDesc(bytesBeingMoved)); + StringUtils.byteDesc(bytesBeingMoved), + nnc.getNameNodeUri()); } } @@ -653,8 +660,10 @@ Result runOneIteration() { System.out.println("No block can be moved. Exiting..."); return newResult(ExitStatus.NO_MOVE_BLOCK, bytesLeftToMove, bytesBeingMoved); } else { - LOG.info( "Will move " + StringUtils.byteDesc(bytesBeingMoved) + - " in this iteration"); + LOG.info("Will move {} in this iteration for {}", + StringUtils.byteDesc(bytesBeingMoved), nnc.toString()); + LOG.info("Total target DataNodes in this iteration: {}", + dispatcher.moveTasksTotal()); } /* For each pair of , start a thread that repeatedly @@ -705,7 +714,9 @@ static private int doBalance(Collection namenodes, LOG.info("excluded nodes = " + p.getExcludedNodes()); LOG.info("source nodes = " + p.getSourceNodes()); checkKeytabAndInit(conf); - System.out.println("Time Stamp Iteration# Bytes Already Moved Bytes Left To Move Bytes Being Moved"); + System.out.println("Time Stamp Iteration#" + + " Bytes Already Moved Bytes Left To Move Bytes Being Moved" + + " NameNode"); List connectors = Collections.emptyList(); try { @@ -721,7 +732,7 @@ static private int doBalance(Collection namenodes, || p.getBlockPools().contains(nnc.getBlockpoolID())) { final Balancer b = new Balancer(nnc, p, conf); final Result r = b.runOneIteration(); - r.print(iteration, System.out); + r.print(iteration, nnc, System.out); // clean all lists b.resetData(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java index c222270882fc0..e19fbeb956fd7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java @@ -83,8 +83,8 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** Dispatching block replica moves between datanodes. */ @InterfaceAudience.Private @@ -392,7 +392,7 @@ private void dispatch() { sendRequest(out, eb, accessToken); receiveResponse(in); - nnc.getBytesMoved().addAndGet(reportedBlock.getNumBytes()); + nnc.addBytesMoved(reportedBlock.getNumBytes()); target.getDDatanode().setHasSuccess(); LOG.info("Successfully moved " + this); } catch (IOException e) { @@ -1064,6 +1064,10 @@ long getBytesMoved() { return nnc.getBytesMoved().get(); } + long getBblocksMoved() { + return nnc.getBlocksMoved().get(); + } + long bytesToMove() { Preconditions.checkState( storageGroupMap.size() >= sources.size() + targets.size(), @@ -1083,6 +1087,14 @@ void add(Source source, StorageGroup target) { targets.add(target); } + public int moveTasksTotal() { + int b = 0; + for (Source src : sources) { + b += src.tasks.size(); + } + return b; + } + private boolean shouldIgnore(DatanodeInfo dn) { // ignore out-of-service nodes final boolean outOfService = !dn.isInService(); @@ -1164,12 +1176,13 @@ public boolean dispatchAndCheckContinue() throws InterruptedException { */ private long dispatchBlockMoves() throws InterruptedException { final long bytesLastMoved = getBytesMoved(); + final long blocksLastMoved = getBblocksMoved(); final Future[] futures = new Future[sources.size()]; int concurrentThreads = Math.min(sources.size(), ((ThreadPoolExecutor)dispatchExecutor).getCorePoolSize()); assert concurrentThreads > 0 : "Number of concurrent threads is 0."; - LOG.debug("Balancer concurrent dispatcher threads = {}", concurrentThreads); + LOG.info("Balancer concurrent dispatcher threads = {}", concurrentThreads); // Determine the size of each mover thread pool per target int threadsPerTarget = maxMoverThreads/targets.size(); @@ -1211,6 +1224,9 @@ public void run() { // wait for all reportedBlock moving to be done waitForMoveCompletion(targets); + LOG.info("Total bytes (blocks) moved in this iteration {} ({})", + StringUtils.byteDesc(getBytesMoved() - bytesLastMoved), + (getBblocksMoved() - blocksLastMoved)); return getBytesMoved() - bytesLastMoved; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java index 586c68048a955..7f54c63303ca6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java @@ -31,8 +31,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.RateLimiter; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.RateLimiter; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HAUtil; @@ -63,7 +63,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.RemoteException; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * The class provides utilities for accessing a NameNode. @@ -162,6 +162,7 @@ public static void checkOtherInstanceRunning(boolean toCheck) { private OutputStream out; private final List targetPaths; private final AtomicLong bytesMoved = new AtomicLong(); + private final AtomicLong blocksMoved = new AtomicLong(); private final int maxNotChangedIterations; private int notChangedIterations = 0; @@ -233,6 +234,19 @@ AtomicLong getBytesMoved() { return bytesMoved; } + AtomicLong getBlocksMoved() { + return blocksMoved; + } + + public void addBytesMoved(long numBytes) { + bytesMoved.addAndGet(numBytes); + blocksMoved.incrementAndGet(); + } + + public URI getNameNodeUri() { + return nameNodeUri; + } + /** @return blocks with locations. */ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, long minBlockSize) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java index ad6fa0f8ad987..6926c3e01c448 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java @@ -27,7 +27,7 @@ import java.util.Random; import java.util.Set; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java index 226487b62aeec..c55329b8e50af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSConfigKeys; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index bec6ec8368120..0ac1d53e14267 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockType; import org.apache.hadoop.hdfs.protocol.HdfsConstants; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java index dc6cf3266a5fc..c6a7bb52b7045 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -22,7 +22,7 @@ import java.util.List; import java.util.NoSuchElementException; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java index 7378e6f21b765..f830678b429b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java index ce186a7b3d1d2..5a133412a85d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockType; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index f2cd6b9819efb..4b1581f450c7b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -126,8 +126,8 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.util.VersionInfo; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -2531,7 +2531,7 @@ void processPendingReconstructions() { * with the most up-to-date block information (e.g. genstamp). */ BlockInfo bi = blocksMap.getStoredBlock(timedOutItems[i]); - if (bi == null) { + if (bi == null || bi.isDeleted()) { continue; } NumberReplicas num = countNodes(timedOutItems[i]); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerFaultInjector.java index 957c5c0c3733c..a529eca6d6ceb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerFaultInjector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerFaultInjector.java @@ -19,7 +19,7 @@ import java.io.IOException; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.server.protocol.BlockReportContext; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java index aecdb59df066f..0e8937548f3fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java @@ -35,8 +35,8 @@ import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.util.Daemon; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -294,65 +294,74 @@ void setBlockTotal(long total) { } String getSafeModeTip() { - String msg = ""; + StringBuilder msg = new StringBuilder(); + boolean isBlockThresholdMet = false; synchronized (this) { - if (blockSafe < blockThreshold) { - msg += String.format( + isBlockThresholdMet = (blockSafe >= blockThreshold); + if (!isBlockThresholdMet) { + msg.append(String.format( "The reported blocks %d needs additional %d" + " blocks to reach the threshold %.4f of total blocks %d.%n", - blockSafe, (blockThreshold - blockSafe), threshold, blockTotal); + blockSafe, (blockThreshold - blockSafe), threshold, blockTotal)); } else { - msg += String.format("The reported blocks %d has reached the threshold" - + " %.4f of total blocks %d. ", blockSafe, threshold, blockTotal); + msg.append(String.format( + "The reported blocks %d has reached the threshold %.4f of total" + + " blocks %d. ", blockSafe, threshold, blockTotal)); } } if (datanodeThreshold > 0) { - int numLive = blockManager.getDatanodeManager().getNumLiveDataNodes(); - if (numLive < datanodeThreshold) { - msg += String.format( - "The number of live datanodes %d needs an additional %d live " - + "datanodes to reach the minimum number %d.%n", - numLive, (datanodeThreshold - numLive), datanodeThreshold); + if (isBlockThresholdMet) { + int numLive = blockManager.getDatanodeManager().getNumLiveDataNodes(); + if (numLive < datanodeThreshold) { + msg.append(String.format( + "The number of live datanodes %d needs an additional %d live " + + "datanodes to reach the minimum number %d.%n", + numLive, (datanodeThreshold - numLive), datanodeThreshold)); + } else { + msg.append(String.format( + "The number of live datanodes %d has reached the minimum number" + + " %d. ", numLive, datanodeThreshold)); + } } else { - msg += String.format("The number of live datanodes %d has reached " - + "the minimum number %d. ", - numLive, datanodeThreshold); + msg.append("The number of live datanodes is not calculated ") + .append("since reported blocks hasn't reached the threshold. "); } } else { - msg += "The minimum number of live datanodes is not required. "; + msg.append("The minimum number of live datanodes is not required. "); } if (getBytesInFuture() > 0) { - msg += "Name node detected blocks with generation stamps " + - "in future. This means that Name node metadata is inconsistent. " + - "This can happen if Name node metadata files have been manually " + - "replaced. Exiting safe mode will cause loss of " + - getBytesInFuture() + " byte(s). Please restart name node with " + - "right metadata or use \"hdfs dfsadmin -safemode forceExit\" " + - "if you are certain that the NameNode was started with the " + - "correct FsImage and edit logs. If you encountered this during " + - "a rollback, it is safe to exit with -safemode forceExit."; - return msg; + msg.append("Name node detected blocks with generation stamps in future. ") + .append("This means that Name node metadata is inconsistent. This ") + .append("can happen if Name node metadata files have been manually ") + .append("replaced. Exiting safe mode will cause loss of ") + .append(getBytesInFuture()) + .append(" byte(s). Please restart name node with right metadata ") + .append("or use \"hdfs dfsadmin -safemode forceExit\" if you ") + .append("are certain that the NameNode was started with the correct ") + .append("FsImage and edit logs. If you encountered this during ") + .append("a rollback, it is safe to exit with -safemode forceExit."); + return msg.toString(); } final String turnOffTip = "Safe mode will be turned off automatically "; switch(status) { case PENDING_THRESHOLD: - msg += turnOffTip + "once the thresholds have been reached."; + msg.append(turnOffTip).append("once the thresholds have been reached."); break; case EXTENSION: - msg += "In safe mode extension. "+ turnOffTip + "in " + - timeToLeaveExtension() / 1000 + " seconds."; + msg.append("In safe mode extension. ").append(turnOffTip).append("in ") + .append(timeToLeaveExtension() / 1000).append(" seconds."); break; case OFF: - msg += turnOffTip + "soon."; + msg.append(turnOffTip).append("soon."); break; default: assert false : "Non-recognized block manager safe mode status: " + status; } - return msg; + return msg.toString(); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java index 563183882765c..07df43b83dcf8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java @@ -25,7 +25,7 @@ import java.util.Map; import java.util.Set; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.AddBlockFlag; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index a1ea63de22fd7..5761690cc3a3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -17,12 +17,14 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYSTORAGETYPE_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYSTORAGETYPE_KEY; import static org.apache.hadoop.util.Time.monotonicNow; import java.util.*; import java.util.concurrent.TimeUnit; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.AddBlockFlag; @@ -35,7 +37,7 @@ import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * The class is responsible for choosing the desired number of targets @@ -92,7 +94,8 @@ private String getText() { } } - protected boolean considerLoad; + protected boolean considerLoad; + private boolean considerLoadByStorageType; protected double considerLoadFactor; private boolean preferLocalNode; protected NetworkTopology clusterMap; @@ -116,6 +119,9 @@ public void initialize(Configuration conf, FSClusterStats stats, this.considerLoad = conf.getBoolean( DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_DEFAULT); + this.considerLoadByStorageType = conf.getBoolean( + DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYSTORAGETYPE_KEY, + DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYSTORAGETYPE_DEFAULT); this.considerLoadFactor = conf.getDouble( DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR, DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR_DEFAULT); @@ -976,8 +982,9 @@ private static void logNodeIsNotChosen(DatanodeDescriptor node, * @return Return true if the datanode should be excluded, otherwise false */ boolean excludeNodeByLoad(DatanodeDescriptor node){ - final double maxLoad = considerLoadFactor * - stats.getInServiceXceiverAverage(); + double inServiceXceiverCount = getInServiceXceiverAverage(node); + final double maxLoad = considerLoadFactor * inServiceXceiverCount; + final int nodeLoad = node.getXceiverCount(); if ((nodeLoad > maxLoad) && (maxLoad > 0)) { logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY, @@ -987,6 +994,48 @@ boolean excludeNodeByLoad(DatanodeDescriptor node){ return false; } + /** + * Gets the inServiceXceiver average count for the cluster, if + * considerLoadByStorageType is true, then load is calculated only for the + * storage types present on the datanode. + * @param node the datanode whose storage types are to be taken into account. + * @return the InServiceXceiverAverage count. + */ + private double getInServiceXceiverAverage(DatanodeDescriptor node) { + double inServiceXceiverCount; + if (considerLoadByStorageType) { + inServiceXceiverCount = + getInServiceXceiverAverageByStorageType(node.getStorageTypes()); + } else { + inServiceXceiverCount = stats.getInServiceXceiverAverage(); + } + return inServiceXceiverCount; + } + + /** + * Gets the average xceiver count with respect to the storage types. + * @param storageTypes the storage types. + * @return the average xceiver count wrt the provided storage types. + */ + private double getInServiceXceiverAverageByStorageType( + Set storageTypes) { + double avgLoad = 0; + final Map storageStats = + stats.getStorageTypeStats(); + int numNodes = 0; + int numXceiver = 0; + for (StorageType s : storageTypes) { + StorageTypeStats storageTypeStats = storageStats.get(s); + numNodes += storageTypeStats.getNodesInService(); + numXceiver += storageTypeStats.getNodesInServiceXceiverCount(); + } + if (numNodes != 0) { + avgLoad = (double) numXceiver / numNodes; + } + + return avgLoad; + } + /** * Determine if a datanode is good for placing block. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java index 2a4b6e8455226..3560a36dde1bc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.util.Time; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java index d62cb727c6c8d..033e95a572297 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; @@ -63,6 +63,12 @@ public static BlockStoragePolicySuite createDefaultSuite( new StorageType[]{StorageType.DISK}, new StorageType[]{StorageType.DISK}, true); // Cannot be changed on regular files, but inherited. + final byte allnvdimmId = HdfsConstants.StoragePolicy.ALL_NVDIMM.value(); + policies[allnvdimmId] = new BlockStoragePolicy(allnvdimmId, + HdfsConstants.StoragePolicy.ALL_NVDIMM.name(), + new StorageType[]{StorageType.NVDIMM}, + new StorageType[]{StorageType.DISK}, + new StorageType[]{StorageType.DISK}); final byte allssdId = HdfsConstants.StoragePolicy.ALL_SSD.value(); policies[allssdId] = new BlockStoragePolicy(allssdId, HdfsConstants.StoragePolicy.ALL_SSD.name(), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockToMarkCorrupt.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockToMarkCorrupt.java index a871390ae2274..3ce5ef07acdca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockToMarkCorrupt.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockToMarkCorrupt.java @@ -19,7 +19,7 @@ import static org.apache.hadoop.hdfs.server.blockmanagement.CorruptReplicasMap.Reason; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.protocol.Block; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java index 35e4a2e92b89b..eab58124cb11c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java @@ -55,7 +55,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Scans the namesystem, scheduling blocks to be cached as appropriate. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CombinedHostFileManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CombinedHostFileManager.java index 9a43014a760e5..662e2e471dfbc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CombinedHostFileManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CombinedHostFileManager.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; -import com.google.common.collect.UnmodifiableIterator; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultimap; +import org.apache.hadoop.thirdparty.com.google.common.collect.Multimap; +import org.apache.hadoop.thirdparty.com.google.common.collect.UnmodifiableIterator; import java.util.stream.Collectors; import org.slf4j.Logger; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java index fc31584ae694c..fdc8bb7491c15 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java @@ -30,7 +30,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.ipc.Server; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Stores information about all corrupt blocks in the File System. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java index af2c12f35c78a..c04f3daabf70e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import com.google.common.collect.Iterables; +import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeFile; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java index a5650d1c4865b..dfa7bf4e8381b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.server.namenode.INode; @@ -40,7 +40,7 @@ /** * Checks to see if datanodes have finished DECOMMISSION_INPROGRESS or * ENTERING_MAINTENANCE state. - *

+ *

* Since this is done while holding the namesystem lock, * the amount of work per monitor tick is limited. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java index 8841d99661e90..8cad44f0ca2fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument; import static org.apache.hadoop.util.Time.monotonicNow; import java.util.Queue; @@ -33,8 +33,8 @@ import org.apache.hadoop.util.ReflectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; /** * Manages decommissioning and maintenance state for DataNodes. A background diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 3fa9b3ad51dd2..2dbcc21775cc0 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -30,7 +30,7 @@ import java.util.Queue; import java.util.Set; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -404,6 +404,7 @@ private void updateStorageStats(StorageReport[] reports, long cacheCapacity, long totalBlockPoolUsed = 0; long totalDfsUsed = 0; long totalNonDfsUsed = 0; + Set visitedMount = new HashSet<>(); Set failedStorageInfos = null; // Decide if we should check for any missing StorageReport and mark it as @@ -472,7 +473,17 @@ private void updateStorageStats(StorageReport[] reports, long cacheCapacity, totalRemaining += report.getRemaining(); totalBlockPoolUsed += report.getBlockPoolUsed(); totalDfsUsed += report.getDfsUsed(); - totalNonDfsUsed += report.getNonDfsUsed(); + String mount = report.getMount(); + // For volumes on the same mount, + // ignore duplicated volumes for nonDfsUsed. + if (mount == null || mount.isEmpty()) { + totalNonDfsUsed += report.getNonDfsUsed(); + } else { + if (!visitedMount.contains(mount)) { + totalNonDfsUsed += report.getNonDfsUsed(); + visitedMount.add(mount); + } + } } // Update total metrics for the node. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index fbe132a066239..01dfe04cb137a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -20,10 +20,11 @@ import static org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol.DNA_ERASURE_CODING_RECONSTRUCTION; import static org.apache.hadoop.util.Time.monotonicNow; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.net.InetAddresses; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses; +import org.apache.hadoop.fs.StorageType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.HadoopIllegalArgumentException; @@ -326,12 +327,14 @@ public class DatanodeManager { this.readConsiderStorageType = conf.getBoolean( DFSConfigKeys.DFS_NAMENODE_READ_CONSIDERSTORAGETYPE_KEY, DFSConfigKeys.DFS_NAMENODE_READ_CONSIDERSTORAGETYPE_DEFAULT); - LOG.warn( - "{} and {} are incompatible and only one can be enabled. " - + "Both are currently enabled.", - DFSConfigKeys.DFS_NAMENODE_READ_CONSIDERLOAD_KEY, - DFSConfigKeys.DFS_NAMENODE_READ_CONSIDERSTORAGETYPE_KEY); - + if (readConsiderLoad && readConsiderStorageType) { + LOG.warn( + "{} and {} are incompatible and only one can be enabled. " + + "Both are currently enabled. {} will be ignored.", + DFSConfigKeys.DFS_NAMENODE_READ_CONSIDERLOAD_KEY, + DFSConfigKeys.DFS_NAMENODE_READ_CONSIDERSTORAGETYPE_KEY, + DFSConfigKeys.DFS_NAMENODE_READ_CONSIDERSTORAGETYPE_KEY); + } this.avoidStaleDataNodesForWrite = conf.getBoolean( DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY, DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_DEFAULT); @@ -1976,6 +1979,11 @@ public double getInServiceXceiverAverage() { } return avgLoad; } + + @Override + public Map getStorageTypeStats() { + return heartbeatManager.getStorageTypeStats(); + } }; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index 3a56ef16c8efd..ee20ada2e8f87 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -30,7 +30,7 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.hdfs.util.FoldedTreeSet; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * A Datanode has one or more storages. A storage in the Datanode is represented diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ExcessRedundancyMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ExcessRedundancyMap.java index ccdcf5451ccd8..41bb7d3428c39 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ExcessRedundancyMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ExcessRedundancyMap.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.slf4j.Logger; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Maps a datnode to the set of excess redundancy details. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/FSClusterStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/FSClusterStats.java index 556b7fcaad73c..14122952bb18d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/FSClusterStats.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/FSClusterStats.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.fs.StorageType; + +import java.util.Map; /** * This interface is used for retrieving the load related statistics of @@ -57,4 +60,10 @@ public interface FSClusterStats { * writes that are currently occurring on the cluster. */ public double getInServiceXceiverAverage(); + + /** + * Indicates the storage statistics per storage type. + * @return storage statistics per storage type. + */ + Map getStorageTypeStats(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java index 9e4d867a0bfc4..5da47c4b2a8a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java @@ -36,7 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Manage the heartbeats received from datanodes. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostFileManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostFileManager.java index 4ead0ba6f704a..57b690262a6d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostFileManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostFileManager.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java index 1d16caef86843..af0c92df5e272 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java @@ -18,10 +18,10 @@ package org.apache.hadoop.hdfs.server.blockmanagement; -import com.google.common.base.Preconditions; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; -import com.google.common.collect.UnmodifiableIterator; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultimap; +import org.apache.hadoop.thirdparty.com.google.common.collect.Multimap; +import org.apache.hadoop.thirdparty.com.google.common.collect.UnmodifiableIterator; import java.net.InetAddress; import java.net.InetSocketAddress; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java index bbe729c3332cc..1ce967a4f3703 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java @@ -39,7 +39,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.hdfs.DFSUtil; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Keeps a Collection for every named machine containing blocks diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java index 8cf9dd40ca6d8..d719e937f2e86 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import java.util.ArrayList; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -382,17 +383,18 @@ boolean remove(BlockInfo block, int priLevel, int oldExpectedReplicas) { } else { // Try to remove the block from all queues if the block was // not found in the queue for the given priority level. + boolean found = false; for (int i = 0; i < LEVEL; i++) { if (i != priLevel && priorityQueues.get(i).remove(block)) { NameNode.blockStateChangeLog.debug( "BLOCK* NameSystem.LowRedundancyBlock.remove: Removing block" + " {} from priority queue {}", block, i); decrementBlockStat(block, i, oldExpectedReplicas); - return true; + found = true; } } + return found; } - return false; } private void decrementBlockStat(BlockInfo blockInfo, int priLevel, @@ -499,6 +501,8 @@ synchronized List> chooseLowRedundancyBlocks( * the block count is met or iteration reaches the end of the lowest priority * list, in which case bookmarks for each block list are reset to the heads * of their respective lists. + * If a block is deleted (has invalid bcId), it will be removed from the low + * redundancy queues. * * @param blocksToProcess - number of blocks to fetch from low redundancy * blocks. @@ -514,21 +518,32 @@ synchronized List> chooseLowRedundancyBlocks( int count = 0; int priority = 0; + HashSet toRemove = new HashSet<>(); for (; count < blocksToProcess && priority < LEVEL; priority++) { - if (priority == QUEUE_WITH_CORRUPT_BLOCKS) { - // do not choose corrupted blocks. - continue; - } - // Go through all blocks that need reconstructions with current priority. // Set the iterator to the first unprocessed block at this priority level + // We do not want to skip QUEUE_WITH_CORRUPT_BLOCKS because we still need + // to look for deleted blocks if any. + final boolean inCorruptLevel = (QUEUE_WITH_CORRUPT_BLOCKS == priority); final Iterator i = priorityQueues.get(priority).getBookmark(); final List blocks = new LinkedList<>(); - blocksToReconstruct.add(blocks); - // Loop through all remaining blocks in the list. + if (!inCorruptLevel) { + blocksToReconstruct.add(blocks); + } for(; count < blocksToProcess && i.hasNext(); count++) { - blocks.add(i.next()); + BlockInfo block = i.next(); + if (block.isDeleted()) { + toRemove.add(block); + continue; + } + if (!inCorruptLevel) { + blocks.add(block); + } + } + for (BlockInfo bInfo : toRemove) { + remove(bInfo, priority); } + toRemove.clear(); } if (priority == LEVEL || resetIterators) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java index 6e9dfa2c0089e..8a8501f4aeee5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java @@ -24,8 +24,8 @@ import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; /** * In the Standby Node, we can receive messages about blocks diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java index 6e1af5729aae5..acf05986397e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.Map; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.util.Daemon; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingRecoveryBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingRecoveryBlocks.java index 3f5f27c819000..8a432cead742b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingRecoveryBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingRecoveryBlocks.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.util.Time; import org.slf4j.Logger; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ProvidedStorageMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ProvidedStorageMap.java index c8c9bb5d1cc74..31ab66d93f6b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ProvidedStorageMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ProvidedStorageMap.java @@ -29,7 +29,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentSkipListMap; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java index 894121eeee7d0..08ebf8e481d22 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java @@ -23,10 +23,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.common.primitives.Doubles; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Doubles; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java index 03a6918152f40..5b30b738c7ab5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java @@ -22,9 +22,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; -import com.google.common.primitives.Ints; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java index c335ec6a7ef89..f90dbad6980ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java @@ -20,6 +20,7 @@ import java.beans.ConstructorProperties; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.StorageType; @@ -39,6 +40,15 @@ public class StorageTypeStats { private int nodesInService = 0; private StorageType storageType; + @VisibleForTesting + void setDataNodesInServiceXceiverCount(int avgXceiverPerDatanode, + int numNodesInService) { + this.nodesInService = numNodesInService; + this.nodesInServiceXceiverCount = numNodesInService * avgXceiverPerDatanode; + } + + private int nodesInServiceXceiverCount; + @ConstructorProperties({"capacityTotal", "capacityUsed", "capacityNonDfsUsed", "capacityRemaining", "blockPoolUsed", "nodesInService"}) public StorageTypeStats( @@ -101,6 +111,10 @@ public int getNodesInService() { return nodesInService; } + public int getNodesInServiceXceiverCount() { + return nodesInServiceXceiverCount; + } + StorageTypeStats(StorageType storageType) { this.storageType = storageType; } @@ -131,6 +145,7 @@ void addStorage(final DatanodeStorageInfo info, void addNode(final DatanodeDescriptor node) { if (node.isInService()) { nodesInService++; + nodesInServiceXceiverCount += node.getXceiverCount(); } } @@ -151,6 +166,7 @@ void subtractStorage(final DatanodeStorageInfo info, void subtractNode(final DatanodeDescriptor node) { if (node.isInService()) { nodesInService--; + nodesInServiceXceiverCount -= node.getXceiverCount(); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index 78d4289a047cb..254e9790da002 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -27,11 +27,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.MetaRecoveryContext; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion; import org.apache.hadoop.util.StringUtils; @@ -61,12 +60,6 @@ public interface HdfsServerConstants { */ int NAMENODE_LAYOUT_VERSION = NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION; - /** - * Current layout version for DataNode. - * Please see {@link DataNodeLayoutVersion.Feature} on adding new layout version. - */ - int DATANODE_LAYOUT_VERSION - = DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION; /** * Path components that are reserved in HDFS. *

@@ -366,6 +359,7 @@ enum BlockUCState { "security.hdfs.unreadable.by.superuser"; String XATTR_ERASURECODING_POLICY = "system.hdfs.erasurecoding.policy"; + String XATTR_SNAPSHOT_DELETED = "system.hdfs.snapshot.deleted"; String XATTR_SATISFY_STORAGE_POLICY = "user.hdfs.sps"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java index e7da44e689a60..ea10f011f9581 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java @@ -53,8 +53,8 @@ import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.util.VersionInfo; -import com.google.common.base.Charsets; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java index 28871e5da32a0..8643821c86993 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion; import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion; -import com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; /** * Common class for storage information. @@ -224,7 +224,8 @@ public void setServiceLayoutVersion(int lv) { } public int getServiceLayoutVersion() { - return storageType == NodeType.DATA_NODE ? HdfsServerConstants.DATANODE_LAYOUT_VERSION + return storageType == NodeType.DATA_NODE + ? DataNodeLayoutVersion.getCurrentLayoutVersion() : HdfsServerConstants.NAMENODE_LAYOUT_VERSION; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java index e9f9bfb830cdc..14cc1c4576d85 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java @@ -36,8 +36,8 @@ import java.util.Map; import java.util.Set; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java index 4d65142c95848..41b5d33785cea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java @@ -54,7 +54,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * This class is used for block maps stored as text files, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index 3199e53890f30..40046b0d8c454 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.StorageType; @@ -206,6 +206,7 @@ String getBlockPoolId(boolean quiet) { if (id != null) { return id; } + DataNodeFaultInjector.get().delayWhenOfferServiceHoldLock(); readLock(); try { if (bpNSInfo != null) { @@ -382,6 +383,7 @@ void verifyAndSetNamespaceInfo(BPServiceActor actor, NamespaceInfo nsInfo) } try { + DataNodeFaultInjector.get().delayWhenOfferServiceHoldLock(); if (setNamespaceInfo(nsInfo) == null) { boolean success = false; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index c4496199f8036..0cc9fac8df7a2 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -71,8 +71,8 @@ import org.apache.hadoop.util.VersionUtil; import org.slf4j.Logger; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; /** * A thread per active or standby namenode to perform: @@ -572,11 +572,11 @@ void start() { } bpThread = new Thread(this); bpThread.setDaemon(true); // needed for JUnit testing - bpThread.start(); if (lifelineSender != null) { lifelineSender.start(); } + bpThread.start(); } private String formatThreadName( @@ -1312,6 +1312,10 @@ public void run() { processQueue(); } catch (Throwable t) { LOG.error("{} encountered fatal exception and exit.", getName(), t); + runningState = RunningState.FAILED; + } finally { + LOG.warn("Ending command processor service for: " + this); + shouldServiceRun = false; } } @@ -1327,6 +1331,7 @@ private void processQueue() { dn.getMetrics().incrNumProcessedCommands(); } catch (InterruptedException e) { LOG.error("{} encountered interrupt and exit.", getName()); + Thread.currentThread().interrupt(); // ignore unless thread was specifically interrupted. if (Thread.interrupted()) { break; @@ -1398,4 +1403,11 @@ void enqueue(DatanodeCommand[] cmds) throws InterruptedException { dn.getMetrics().incrActorCmdQueueLength(1); } } + + @VisibleForTesting + void stopCommandProcessingThread() { + if (commandProcessingThread != null) { + commandProcessingThread.interrupt(); + } + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java index 3388855f8fbdf..634fcdd012178 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.DFSUtilClient; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java index 9a7b6bcf7e52e..390577f8b54bc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java @@ -29,11 +29,11 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.security.UserGroupInformation; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import org.slf4j.Logger; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java index 539baf114d6fe..d2da2c5678924 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java @@ -38,7 +38,6 @@ import org.apache.hadoop.fs.HardLink; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.LayoutVersion; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; @@ -47,9 +46,9 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.util.Daemon; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * Manages storage for the set of BlockPoolSlices which share a particular @@ -285,7 +284,7 @@ private void format(StorageDirectory bpSdir, NamespaceInfo nsInfo) throws IOExce LOG.info("Formatting block pool {} directory {}", blockpoolID, bpSdir .getCurrentDir()); bpSdir.clearDirectory(); // create directory - this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; + this.layoutVersion = DataNodeLayoutVersion.getCurrentLayoutVersion(); this.cTime = nsInfo.getCTime(); this.namespaceID = nsInfo.getNamespaceID(); this.blockpoolID = nsInfo.getBlockPoolID(); @@ -388,7 +387,7 @@ private boolean doTransition(StorageDirectory sd, NamespaceInfo nsInfo, } readProperties(sd); checkVersionUpgradable(this.layoutVersion); - assert this.layoutVersion >= HdfsServerConstants.DATANODE_LAYOUT_VERSION + assert this.layoutVersion >= DataNodeLayoutVersion.getCurrentLayoutVersion() : "Future version is not allowed"; if (getNamespaceID() != nsInfo.getNamespaceID()) { throw new IOException("Incompatible namespaceIDs in " @@ -402,17 +401,17 @@ private boolean doTransition(StorageDirectory sd, NamespaceInfo nsInfo, + nsInfo.getBlockPoolID() + "; datanode blockpoolID = " + blockpoolID); } - if (this.layoutVersion == HdfsServerConstants.DATANODE_LAYOUT_VERSION + if (this.layoutVersion == DataNodeLayoutVersion.getCurrentLayoutVersion() && this.cTime == nsInfo.getCTime()) { return false; // regular startup } - if (this.layoutVersion > HdfsServerConstants.DATANODE_LAYOUT_VERSION) { + if (this.layoutVersion > DataNodeLayoutVersion.getCurrentLayoutVersion()) { int restored = restoreBlockFilesFromTrash(getTrashRootDir(sd)); LOG.info("Restored {} block files from trash " + "before the layout upgrade. These blocks will be moved to " + "the previous directory during the upgrade", restored); } - if (this.layoutVersion > HdfsServerConstants.DATANODE_LAYOUT_VERSION + if (this.layoutVersion > DataNodeLayoutVersion.getCurrentLayoutVersion() || this.cTime < nsInfo.getCTime()) { doUpgrade(sd, nsInfo, callables, conf); // upgrade return true; @@ -459,8 +458,8 @@ private void doUpgrade(final StorageDirectory bpSd, final int oldLV = getLayoutVersion(); LOG.info("Upgrading block pool storage directory {}.\n old LV = {}; old" + " CTime = {}.\n new LV = {}; new CTime = {}", - bpSd.getRoot(), oldLV, this.getCTime(), HdfsServerConstants - .DATANODE_LAYOUT_VERSION, nsInfo.getCTime()); + bpSd.getRoot(), oldLV, this.getCTime(), + DataNodeLayoutVersion.getCurrentLayoutVersion(), nsInfo.getCTime()); // get /previous directory String dnRoot = getDataNodeStorageRoot(bpSd.getRoot().getCanonicalPath()); StorageDirectory dnSdStorage = new StorageDirectory(new File(dnRoot)); @@ -507,7 +506,7 @@ private void doUpgrade(String name, final StorageDirectory bpSd, throws IOException { // 3. Create new /current with block files hardlinks and VERSION linkAllBlocks(bpTmpDir, bpCurDir, oldLV, conf); - this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; + this.layoutVersion = DataNodeLayoutVersion.getCurrentLayoutVersion(); assert this.namespaceID == nsInfo.getNamespaceID() : "Data-node and name-node layout versions must be the same."; this.cTime = nsInfo.getCTime(); @@ -616,13 +615,15 @@ void doRollback(StorageDirectory bpSd, NamespaceInfo nsInfo) // the namespace state or can be further upgraded to it. // In another word, we can only roll back when ( storedLV >= software LV) // && ( DN.previousCTime <= NN.ctime) - if (!(prevInfo.getLayoutVersion() >= HdfsServerConstants.DATANODE_LAYOUT_VERSION && + if (!(prevInfo.getLayoutVersion() >= + DataNodeLayoutVersion.getCurrentLayoutVersion() && prevInfo.getCTime() <= nsInfo.getCTime())) { // cannot rollback throw new InconsistentFSStateException(bpSd.getRoot(), "Cannot rollback to a newer state.\nDatanode previous state: LV = " + prevInfo.getLayoutVersion() + " CTime = " + prevInfo.getCTime() + " is newer than the namespace state: LV = " - + HdfsServerConstants.DATANODE_LAYOUT_VERSION + " CTime = " + nsInfo.getCTime()); + + DataNodeLayoutVersion.getCurrentLayoutVersion() + " CTime = " + + nsInfo.getCTime()); } LOG.info("Rolling back storage directory {}.\n target LV = {}; target " diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index a153d492ce00d..b6970020b934c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -64,7 +64,7 @@ import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED; import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.SYNC_FILE_RANGE_WRITE; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; /** A class that receives a block and writes to its own disk, meanwhile @@ -409,7 +409,7 @@ boolean packetSentInTime() { * Flush block data and metadata files to disk. * @throws IOException */ - void flushOrSync(boolean isSync) throws IOException { + void flushOrSync(boolean isSync, long seqno) throws IOException { long flushTotalNanos = 0; long begin = Time.monotonicNow(); if (checksumOut != null) { @@ -448,7 +448,8 @@ void flushOrSync(boolean isSync) throws IOException { LOG.warn("Slow flushOrSync took " + duration + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), isSync:" + isSync + ", flushTotalNanos=" + flushTotalNanos + "ns, volume=" + getVolumeBaseUri() - + ", blockId=" + replicaInfo.getBlockId()); + + ", blockId=" + replicaInfo.getBlockId() + + ", seqno=" + seqno); } } @@ -606,7 +607,8 @@ private int receivePacket() throws IOException { LOG.warn("Slow BlockReceiver write packet to mirror took " + duration + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), " + "downstream DNs=" + Arrays.toString(downstreamDNs) - + ", blockId=" + replicaInfo.getBlockId()); + + ", blockId=" + replicaInfo.getBlockId() + + ", seqno=" + seqno); } } catch (IOException e) { handleMirrorOutError(e); @@ -622,7 +624,7 @@ private int receivePacket() throws IOException { } // sync block if requested if (syncBlock) { - flushOrSync(true); + flushOrSync(true, seqno); } } else { final int checksumLen = diskChecksum.getChecksumSize(len); @@ -741,7 +743,8 @@ private int receivePacket() throws IOException { LOG.warn("Slow BlockReceiver write data to disk cost:" + duration + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), " + "volume=" + getVolumeBaseUri() - + ", blockId=" + replicaInfo.getBlockId()); + + ", blockId=" + replicaInfo.getBlockId() + + ", seqno=" + seqno); } if (duration > maxWriteToDiskMs) { @@ -814,14 +817,14 @@ private int receivePacket() throws IOException { } /// flush entire packet, sync if requested - flushOrSync(syncBlock); + flushOrSync(syncBlock, seqno); replicaInfo.setLastChecksumAndDataLen(offsetInBlock, lastCrc); datanode.metrics.incrBytesWritten(len); datanode.metrics.incrTotalWriteTime(duration); - manageWriterOsCache(offsetInBlock); + manageWriterOsCache(offsetInBlock, seqno); } } catch (IOException iex) { // Volume error check moved to FileIoProvider @@ -887,7 +890,7 @@ private static byte[] copyLastChunkChecksum(byte[] array, int size, int end) { return Arrays.copyOfRange(array, end - size, end); } - private void manageWriterOsCache(long offsetInBlock) { + private void manageWriterOsCache(long offsetInBlock, long seqno) { try { if (streams.getOutFd() != null && offsetInBlock > lastCacheManagementOffset + CACHE_DROP_LAG_BYTES) { @@ -934,7 +937,8 @@ private void manageWriterOsCache(long offsetInBlock) { LOG.warn("Slow manageWriterOsCache took " + duration + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), volume=" + getVolumeBaseUri() - + ", blockId=" + replicaInfo.getBlockId()); + + ", blockId=" + replicaInfo.getBlockId() + + ", seqno=" + seqno); } } } catch (Throwable t) { @@ -1646,7 +1650,8 @@ private void sendAckUpstreamUnprotected(PipelineAck ack, long seqno, + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), " + myString + ", replyAck=" + replyAck + ", downstream DNs=" + Arrays.toString(downstreamDNs) - + ", blockId=" + replicaInfo.getBlockId()); + + ", blockId=" + replicaInfo.getBlockId() + + ", seqno=" + seqno); } else if (LOG.isDebugEnabled()) { LOG.debug(myString + ", replyAck=" + replyAck); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java index f1c396f5b3e81..dfc6f7f13364e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.protocol.DatanodeID; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java index 82efcf86432f9..485cf00152a47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java @@ -22,6 +22,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SCANNER_SKIP_RECENT_ACCESSED_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SCANNER_VOLUME_JOIN_TIMEOUT_MSEC_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SCANNER_VOLUME_JOIN_TIMEOUT_MSEC_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT; @@ -30,11 +32,11 @@ import java.util.TreeMap; import java.util.concurrent.TimeUnit; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.datanode.VolumeScanner.ScanResultHandler; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; @@ -68,6 +70,12 @@ public class BlockScanner { */ private Conf conf; + /** + * Timeout duration in milliseconds waiting for {@link VolumeScanner} to stop + * inside {@link #removeAllVolumeScanners}. + */ + private long joinVolumeScannersTimeOutMs; + @VisibleForTesting void setConf(Conf conf) { this.conf = conf; @@ -185,6 +193,9 @@ public BlockScanner(DataNode datanode) { public BlockScanner(DataNode datanode, Configuration conf) { this.datanode = datanode; + setJoinVolumeScannersTimeOutMs( + conf.getLong(DFS_BLOCK_SCANNER_VOLUME_JOIN_TIMEOUT_MSEC_KEY, + DFS_BLOCK_SCANNER_VOLUME_JOIN_TIMEOUT_MSEC_DEFAULT)); this.conf = new Conf(conf); if (isEnabled()) { LOG.info("Initialized block scanner with targetBytesPerSec {}", @@ -204,6 +215,13 @@ public boolean isEnabled() { return (conf.scanPeriodMs > 0) && (conf.targetBytesPerSec > 0); } + /** + * Returns true if there is any scanner thread registered. + */ + public synchronized boolean hasAnyRegisteredScanner() { + return !scanners.isEmpty(); + } + /** * Set up a scanner for the given block pool and volume. * @@ -268,7 +286,10 @@ public synchronized void removeVolumeScanner(FsVolumeSpi volume) { /** * Stops and removes all volume scanners. * - * This function will block until all the volume scanners have stopped. + * This function is called on shutdown. It will return even if some of + * the scanners don't terminate in time. Since the scanners are daemon + * threads and do not alter the block content, it is safe to ignore + * such conditions on shutdown. */ public synchronized void removeAllVolumeScanners() { for (Entry entry : scanners.entrySet()) { @@ -276,7 +297,7 @@ public synchronized void removeAllVolumeScanners() { } for (Entry entry : scanners.entrySet()) { Uninterruptibles.joinUninterruptibly(entry.getValue(), - 5, TimeUnit.MINUTES); + getJoinVolumeScannersTimeOutMs(), TimeUnit.MILLISECONDS); } scanners.clear(); } @@ -352,6 +373,14 @@ synchronized void markSuspectBlock(String storageId, ExtendedBlock block) { scanner.markSuspectBlock(block); } + public long getJoinVolumeScannersTimeOutMs() { + return joinVolumeScannersTimeOutMs; + } + + public void setJoinVolumeScannersTimeOutMs(long joinScannersTimeOutMs) { + this.joinVolumeScannersTimeOutMs = joinScannersTimeOutMs; + } + @InterfaceAudience.Private public static class Servlet extends HttpServlet { private static final long serialVersionUID = 1L; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java index b396bf970523c..99b9d64e80d41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java @@ -55,8 +55,8 @@ import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED; import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_SEQUENTIAL; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 7f740c39d8cf2..933a15faaaa29 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -222,13 +222,13 @@ import org.apache.htrace.core.Tracer; import org.eclipse.jetty.util.ajax.JSON; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; +import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.thirdparty.protobuf.BlockingService; import org.slf4j.Logger; @@ -1579,7 +1579,7 @@ DatanodeRegistration createBPRegistration(NamespaceInfo nsInfo) { if (storageInfo == null) { // it's null in the case of SimulatedDataSet storageInfo = new StorageInfo( - DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION, + DataNodeLayoutVersion.getCurrentLayoutVersion(), nsInfo.getNamespaceID(), nsInfo.clusterID, nsInfo.getCTime(), NodeType.DATA_NODE); } @@ -1660,7 +1660,9 @@ void shutdownBlockPool(BPOfferService bpos) { // a block pool id String bpId = bpos.getBlockPoolId(); - blockScanner.disableBlockPoolId(bpId); + if (blockScanner.hasAnyRegisteredScanner()) { + blockScanner.disableBlockPoolId(bpId); + } if (data != null) { data.shutdownBlockPool(bpId); @@ -2710,7 +2712,11 @@ void closeBlock(ExtendedBlock block, String delHint, String storageUuid, * If this thread is specifically interrupted, it will stop waiting. */ public void runDatanodeDaemon() throws IOException { - blockPoolManager.startAll(); + + // Verify that blockPoolManager has been started. + if (!isDatanodeUp()) { + throw new IOException("Failed to instantiate DataNode."); + } // start dataXceiveServer dataXceiverServer.start(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java index c031f6c4c57e9..73fa8f847ffa7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java index abe7b742f93e1..ceccbd98a0e67 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java @@ -26,15 +26,36 @@ import org.apache.hadoop.hdfs.protocol.LayoutVersion.FeatureInfo; import org.apache.hadoop.hdfs.protocol.LayoutVersion.LayoutFeature; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; + @InterfaceAudience.Private public class DataNodeLayoutVersion { /** Build layout version and corresponding feature matrix */ public final static Map> FEATURES = new HashMap>(); - public static final int CURRENT_LAYOUT_VERSION + private static int currentLayoutVersion = LayoutVersion.getCurrentLayoutVersion(Feature.values()); + /** + * Method for testing rolling upgrade. + * Do not call this method from production. + * + * @param lv new layout version to set + */ + @VisibleForTesting + static void setCurrentLayoutVersionForTesting(int lv) { + currentLayoutVersion = lv; + } + + /** + * Get current layout version of the DataNode. + * @return the current layout version of the DataNode + */ + public static int getCurrentLayoutVersion() { + return currentLayoutVersion; + } + static{ LayoutVersion.updateMap(FEATURES, LayoutVersion.Feature.values()); LayoutVersion.updateMap(FEATURES, DataNodeLayoutVersion.Feature.values()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index b7faecb1ad599..4e216db892f67 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -53,7 +53,6 @@ import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.LayoutVersion; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; @@ -62,10 +61,10 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.util.Daemon; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ComparisonChain; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; /** * Data storage information file. @@ -571,7 +570,7 @@ void recoverTransitionRead(DataNode datanode, NamespaceInfo nsInfo, void format(StorageDirectory sd, NamespaceInfo nsInfo, String newDatanodeUuid, Configuration conf) throws IOException { sd.clearDirectory(); // create directory - this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; + this.layoutVersion = DataNodeLayoutVersion.getCurrentLayoutVersion(); this.clusterID = nsInfo.getClusterID(); this.namespaceID = nsInfo.getNamespaceID(); this.cTime = 0; @@ -728,8 +727,9 @@ private boolean doTransition(StorageDirectory sd, NamespaceInfo nsInfo, } readProperties(sd); checkVersionUpgradable(this.layoutVersion); - assert this.layoutVersion >= HdfsServerConstants.DATANODE_LAYOUT_VERSION : - "Future version is not allowed"; + assert this.layoutVersion >= + DataNodeLayoutVersion.getCurrentLayoutVersion() : + "Future version is not allowed"; boolean federationSupported = DataNodeLayoutVersion.supports( @@ -752,13 +752,13 @@ private boolean doTransition(StorageDirectory sd, NamespaceInfo nsInfo, } // regular start up. - if (this.layoutVersion == HdfsServerConstants.DATANODE_LAYOUT_VERSION) { + if (this.layoutVersion == DataNodeLayoutVersion.getCurrentLayoutVersion()) { createStorageID(sd, layoutVersion, conf); return false; // need to write properties } // do upgrade - if (this.layoutVersion > HdfsServerConstants.DATANODE_LAYOUT_VERSION) { + if (this.layoutVersion > DataNodeLayoutVersion.getCurrentLayoutVersion()) { if (federationSupported) { // If the existing on-disk layout version supports federation, // simply update the properties. @@ -775,7 +775,7 @@ private boolean doTransition(StorageDirectory sd, NamespaceInfo nsInfo, // failed. throw new IOException("BUG: The stored LV = " + this.getLayoutVersion() + " is newer than the supported LV = " - + HdfsServerConstants.DATANODE_LAYOUT_VERSION); + + DataNodeLayoutVersion.getCurrentLayoutVersion()); } /** @@ -807,7 +807,7 @@ void doUpgradePreFederation(final StorageDirectory sd, final int oldLV = getLayoutVersion(); LOG.info("Upgrading storage directory {}.\n old LV = {}; old CTime = {}" + ".\n new LV = {}; new CTime = {}", sd.getRoot(), oldLV, - this.getCTime(), HdfsServerConstants.DATANODE_LAYOUT_VERSION, + this.getCTime(), DataNodeLayoutVersion.getCurrentLayoutVersion(), nsInfo.getCTime()); final File curDir = sd.getCurrentDir(); @@ -868,9 +868,9 @@ void upgradeProperties(StorageDirectory sd, Configuration conf) throws IOException { createStorageID(sd, layoutVersion, conf); LOG.info("Updating layout version from {} to {} for storage {}", - layoutVersion, HdfsServerConstants.DATANODE_LAYOUT_VERSION, + layoutVersion, DataNodeLayoutVersion.getCurrentLayoutVersion(), sd.getRoot()); - layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; + layoutVersion = DataNodeLayoutVersion.getCurrentLayoutVersion(); writeProperties(sd); } @@ -922,11 +922,11 @@ void doRollback( StorageDirectory sd, // This is a regular startup or a post-federation rollback if (!prevDir.exists()) { if (DataNodeLayoutVersion.supports(LayoutVersion.Feature.FEDERATION, - HdfsServerConstants.DATANODE_LAYOUT_VERSION)) { - readProperties(sd, HdfsServerConstants.DATANODE_LAYOUT_VERSION); + DataNodeLayoutVersion.getCurrentLayoutVersion())) { + readProperties(sd, DataNodeLayoutVersion.getCurrentLayoutVersion()); writeProperties(sd); LOG.info("Layout version rolled back to {} for storage {}", - HdfsServerConstants.DATANODE_LAYOUT_VERSION, sd.getRoot()); + DataNodeLayoutVersion.getCurrentLayoutVersion(), sd.getRoot()); } return; } @@ -935,17 +935,19 @@ void doRollback( StorageDirectory sd, // We allow rollback to a state, which is either consistent with // the namespace state or can be further upgraded to it. - if (!(prevInfo.getLayoutVersion() >= HdfsServerConstants.DATANODE_LAYOUT_VERSION - && prevInfo.getCTime() <= nsInfo.getCTime())) // cannot rollback + if (!(prevInfo.getLayoutVersion() >= + DataNodeLayoutVersion.getCurrentLayoutVersion() + && prevInfo.getCTime() <= nsInfo.getCTime())) { // cannot rollback throw new InconsistentFSStateException(sd.getRoot(), "Cannot rollback to a newer state.\nDatanode previous state: LV = " + prevInfo.getLayoutVersion() + " CTime = " + prevInfo.getCTime() + " is newer than the namespace state: LV = " - + HdfsServerConstants.DATANODE_LAYOUT_VERSION + " CTime = " + + DataNodeLayoutVersion.getCurrentLayoutVersion() + " CTime = " + nsInfo.getCTime()); + } LOG.info("Rolling back storage directory {}.\n target LV = {}; target " + "CTime = {}", sd.getRoot(), - HdfsServerConstants.DATANODE_LAYOUT_VERSION, nsInfo.getCTime()); + DataNodeLayoutVersion.getCurrentLayoutVersion(), nsInfo.getCTime()); File tmpDir = sd.getRemovedTmp(); assert !tmpDir.exists() : "removed.tmp directory must not exist."; // rename current to tmp diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index 9c885fc4ab255..7df5caf60a0ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.thirdparty.protobuf.ByteString; import javax.crypto.SecretKey; import org.apache.commons.logging.Log; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java index 7faf2108bd4fa..b953760ea9b30 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java @@ -35,8 +35,8 @@ import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.util.Daemon; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; @@ -188,6 +188,9 @@ void release() { this.maxXceiverCount = conf.getInt(DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY, DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT); + Preconditions.checkArgument(this.maxXceiverCount >= 1, + DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY + + " should not be less than 1."); this.estimateBlockSize = conf.getLongBytes(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java index b2e521c695a1e..d83510816335f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java @@ -46,15 +46,14 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.ScanInfo; -import org.apache.hadoop.util.AutoCloseableLock; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.StopWatch; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ListMultimap; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.ArrayListMultimap; +import org.apache.hadoop.thirdparty.com.google.common.collect.ListMultimap; /** * Periodically scans the data directories for block and block metadata files. @@ -472,88 +471,85 @@ private void scan() { // Pre-sort the reports outside of the lock blockPoolReport.sortBlocks(); - // Hold FSDataset lock to prevent further changes to the block map - try (AutoCloseableLock lock = dataset.acquireDatasetReadLock()) { - for (final String bpid : blockPoolReport.getBlockPoolIds()) { - List blockpoolReport = blockPoolReport.getScanInfo(bpid); - - Stats statsRecord = new Stats(bpid); - stats.put(bpid, statsRecord); - Collection diffRecord = new ArrayList<>(); - - statsRecord.totalBlocks = blockpoolReport.size(); - final List bl = dataset.getFinalizedBlocks(bpid); - Collections.sort(bl); // Sort based on blockId - - int d = 0; // index for blockpoolReport - int m = 0; // index for memReprot - while (m < bl.size() && d < blockpoolReport.size()) { - ReplicaInfo memBlock = bl.get(m); - ScanInfo info = blockpoolReport.get(d); - if (info.getBlockId() < memBlock.getBlockId()) { - if (!dataset.isDeletingBlock(bpid, info.getBlockId())) { - // Block is missing in memory - statsRecord.missingMemoryBlocks++; - addDifference(diffRecord, statsRecord, info); - } - d++; - continue; - } - if (info.getBlockId() > memBlock.getBlockId()) { - // Block is missing on the disk - addDifference(diffRecord, statsRecord, memBlock.getBlockId(), - info.getVolume()); - m++; - continue; - } - // Block file and/or metadata file exists on the disk - // Block exists in memory - if (info.getVolume().getStorageType() != StorageType.PROVIDED - && info.getBlockFile() == null) { - // Block metadata file exits and block file is missing - addDifference(diffRecord, statsRecord, info); - } else if (info.getGenStamp() != memBlock.getGenerationStamp() - || info.getBlockLength() != memBlock.getNumBytes()) { - // Block metadata file is missing or has wrong generation stamp, - // or block file length is different than expected - statsRecord.mismatchBlocks++; - addDifference(diffRecord, statsRecord, info); - } else if (memBlock.compareWith(info) != 0) { - // volumeMap record and on-disk files do not match. - statsRecord.duplicateBlocks++; + for (final String bpid : blockPoolReport.getBlockPoolIds()) { + List blockpoolReport = blockPoolReport.getScanInfo(bpid); + + Stats statsRecord = new Stats(bpid); + stats.put(bpid, statsRecord); + Collection diffRecord = new ArrayList<>(); + + statsRecord.totalBlocks = blockpoolReport.size(); + final List bl; + bl = dataset.getSortedFinalizedBlocks(bpid); + + int d = 0; // index for blockpoolReport + int m = 0; // index for memReprot + while (m < bl.size() && d < blockpoolReport.size()) { + ReplicaInfo memBlock = bl.get(m); + ScanInfo info = blockpoolReport.get(d); + if (info.getBlockId() < memBlock.getBlockId()) { + if (!dataset.isDeletingBlock(bpid, info.getBlockId())) { + // Block is missing in memory + statsRecord.missingMemoryBlocks++; addDifference(diffRecord, statsRecord, info); } d++; - - if (d < blockpoolReport.size()) { - // There may be multiple on-disk records for the same block, do not - // increment the memory record pointer if so. - ScanInfo nextInfo = blockpoolReport.get(d); - if (nextInfo.getBlockId() != info.getBlockId()) { - ++m; - } - } else { - ++m; - } + continue; } - while (m < bl.size()) { - ReplicaInfo current = bl.get(m++); - addDifference(diffRecord, statsRecord, current.getBlockId(), - current.getVolume()); + if (info.getBlockId() > memBlock.getBlockId()) { + // Block is missing on the disk + addDifference(diffRecord, statsRecord, memBlock.getBlockId(), + info.getVolume()); + m++; + continue; } - while (d < blockpoolReport.size()) { - if (!dataset.isDeletingBlock(bpid, - blockpoolReport.get(d).getBlockId())) { - statsRecord.missingMemoryBlocks++; - addDifference(diffRecord, statsRecord, blockpoolReport.get(d)); + // Block file and/or metadata file exists on the disk + // Block exists in memory + if (info.getVolume().getStorageType() != StorageType.PROVIDED + && info.getBlockFile() == null) { + // Block metadata file exits and block file is missing + addDifference(diffRecord, statsRecord, info); + } else if (info.getGenStamp() != memBlock.getGenerationStamp() + || info.getBlockLength() != memBlock.getNumBytes()) { + // Block metadata file is missing or has wrong generation stamp, + // or block file length is different than expected + statsRecord.mismatchBlocks++; + addDifference(diffRecord, statsRecord, info); + } else if (memBlock.compareWith(info) != 0) { + // volumeMap record and on-disk files do not match. + statsRecord.duplicateBlocks++; + addDifference(diffRecord, statsRecord, info); + } + d++; + + if (d < blockpoolReport.size()) { + // There may be multiple on-disk records for the same block, do not + // increment the memory record pointer if so. + ScanInfo nextInfo = blockpoolReport.get(d); + if (nextInfo.getBlockId() != info.getBlockId()) { + ++m; } - d++; + } else { + ++m; } - synchronized (diffs) { - diffs.addAll(bpid, diffRecord); + } + while (m < bl.size()) { + ReplicaInfo current = bl.get(m++); + addDifference(diffRecord, statsRecord, current.getBlockId(), + current.getVolume()); + } + while (d < blockpoolReport.size()) { + if (!dataset.isDeletingBlock(bpid, + blockpoolReport.get(d).getBlockId())) { + statsRecord.missingMemoryBlocks++; + addDifference(diffRecord, statsRecord, blockpoolReport.get(d)); } - LOG.info("Scan Results: {}", statsRecord); + d++; + } + synchronized (diffs) { + diffs.addAll(bpid, diffRecord); } + LOG.info("Scan Results: {}", statsRecord); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java index ac10e8f249f2e..83180035fdb6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java @@ -18,8 +18,8 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.codec.digest.DigestUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -902,7 +902,7 @@ private long getMaxError(DiskBalancerWorkItem item) { */ private ExtendedBlock getBlockToCopy(FsVolumeSpi.BlockIterator iter, DiskBalancerWorkItem item) { - while (!iter.atEnd() && item.getErrorCount() < getMaxError(item)) { + while (!iter.atEnd() && item.getErrorCount() <= getMaxError(item)) { try { ExtendedBlock block = iter.nextBlock(); if(null == block){ @@ -923,7 +923,7 @@ private ExtendedBlock getBlockToCopy(FsVolumeSpi.BlockIterator iter, item.incErrorCount(); } } - if (item.getErrorCount() >= getMaxError(item)) { + if (item.getErrorCount() > getMaxError(item)) { item.setErrMsg("Error count exceeded."); LOG.info("Maximum error count exceeded. Error count: {} Max error:{} ", item.getErrorCount(), item.getMaxDiskErrors()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java index 73cc44b31f9cf..f55b8c2b73425 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java @@ -37,8 +37,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; /** * Manage Incremental Block Reports (IBRs). diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplica.java index cb53a64f7defe..b711e1a8f2115 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplica.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplica.java @@ -42,7 +42,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * This class is used for all replicas which are on local storage media diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProvidedReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProvidedReplica.java index bd23021f5cdef..00640f62c8428 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProvidedReplica.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProvidedReplica.java @@ -23,7 +23,7 @@ import java.io.OutputStream; import java.net.URI; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.input.BoundedInputStream; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java index 84fac578d9798..6c666411c9671 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java @@ -16,7 +16,7 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.daemon.Daemon; import org.apache.commons.daemon.DaemonContext; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java index 4b35109326d88..cb8dfaf262334 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java @@ -30,7 +30,7 @@ import java.util.Iterator; import java.util.Set; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,9 +46,9 @@ import org.apache.hadoop.net.unix.DomainSocketWatcher; import org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.HashMultimap; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultimap; /** * Manages client short-circuit memory segments on the DataNode. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java index 5e3d523cde382..6bc25eb24a675 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java @@ -30,10 +30,10 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.cache.Cache; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -670,12 +670,14 @@ public void run() { LOG.error("{} exiting because of exception ", this, e); } LOG.info("{} exiting.", this); + VolumeScannerCBInjector.get().preSavingBlockIteratorTask(this); // Save the current position of all block iterators and close them. for (BlockIterator iter : blockIters) { saveBlockIterator(iter); IOUtils.cleanup(null, iter); } } finally { + VolumeScannerCBInjector.get().terminationCallBack(this); // When the VolumeScanner exits, release the reference we were holding // on the volume. This will allow the volume to be removed later. IOUtils.cleanup(null, ref); @@ -695,6 +697,7 @@ public synchronized void shutdown() { stopping = true; notify(); this.interrupt(); + VolumeScannerCBInjector.get().shutdownCallBack(this); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScannerCBInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScannerCBInjector.java new file mode 100644 index 0000000000000..d15d8d45d5e60 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScannerCBInjector.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode; + +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; + +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Used for injecting call backs in {@link VolumeScanner} + * and {@link BlockScanner} tests. + * Calls into this are a no-op in production code. + */ +@VisibleForTesting +@InterfaceAudience.Private +public class VolumeScannerCBInjector { + private static VolumeScannerCBInjector instance = + new VolumeScannerCBInjector(); + + public static VolumeScannerCBInjector get() { + return instance; + } + + public static void set(VolumeScannerCBInjector injector) { + instance = injector; + } + + public void preSavingBlockIteratorTask(final VolumeScanner volumeScanner) { + } + + public void shutdownCallBack(final VolumeScanner volumeScanner) { + } + + public void terminationCallBack(final VolumeScanner volumeScanner) { + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java index 06867fbceb793..6ae4f01299ceb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java @@ -20,15 +20,15 @@ */ package org.apache.hadoop.hdfs.server.datanode.checker; -import com.google.common.annotations.Beta; -import com.google.common.annotations.GwtCompatible; -import com.google.common.base.Preconditions; -import static com.google.common.base.Preconditions.checkNotNull; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; -import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.hadoop.thirdparty.com.google.common.annotations.Beta; +import org.apache.hadoop.thirdparty.com.google.common.annotations.GwtCompatible; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.SettableFuture; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import static java.util.concurrent.atomic.AtomicReferenceFieldUpdater .newUpdater; @@ -55,7 +55,7 @@ * include instantiating a {@link SettableFuture}, submitting a task to a * {@link ListeningExecutorService}, and deriving a {@code Future} from an * existing one, typically using methods like {@link Futures#transform - * (ListenableFuture, com.google.common.base.Function) Futures.transform} + * (ListenableFuture, org.apache.hadoop.thirdparty.com.google.common.base.Function) Futures.transform} * and its overloaded versions. *

*

This class implements all methods in {@code ListenableFuture}. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AsyncChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AsyncChecker.java index d69845448b7aa..75b0ebea389cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AsyncChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AsyncChecker.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.datanode.checker; import java.util.Optional; -import com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java index 91582fe0558a8..d077d215a5750 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java @@ -18,14 +18,14 @@ package org.apache.hadoop.hdfs.server.datanode.checker; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.conf.Configuration; @@ -354,23 +354,29 @@ private class ResultHandler } @Override - public void onSuccess(@Nonnull VolumeCheckResult result) { - switch(result) { - case HEALTHY: - case DEGRADED: - LOG.debug("Volume {} is {}.", reference.getVolume(), result); - markHealthy(); - break; - case FAILED: - LOG.warn("Volume {} detected as being unhealthy", + public void onSuccess(VolumeCheckResult result) { + if (result == null) { + LOG.error("Unexpected health check result null for volume {}", reference.getVolume()); - markFailed(); - break; - default: - LOG.error("Unexpected health check result {} for volume {}", - result, reference.getVolume()); markHealthy(); - break; + } else { + switch(result) { + case HEALTHY: + case DEGRADED: + LOG.debug("Volume {} is {}.", reference.getVolume(), result); + markHealthy(); + break; + case FAILED: + LOG.warn("Volume {} detected as being unhealthy", + reference.getVolume()); + markFailed(); + break; + default: + LOG.error("Unexpected health check result {} for volume {}", + result, reference.getVolume()); + markHealthy(); + break; + } } cleanup(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/StorageLocationChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/StorageLocationChecker.java index 0332bc8633950..3d49dd5362864 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/StorageLocationChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/StorageLocationChecker.java @@ -20,9 +20,9 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.*; -import com.google.common.collect.Maps; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java index 6f04129b4928c..f969c7ade288b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java @@ -18,11 +18,11 @@ package org.apache.hadoop.hdfs.server.datanode.checker; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.util.Timer; @@ -166,7 +166,7 @@ private void addResultCachingCallback( Checkable target, ListenableFuture lf) { Futures.addCallback(lf, new FutureCallback() { @Override - public void onSuccess(@Nullable V result) { + public void onSuccess(V result) { synchronized (ThrottledAsyncChecker.this) { checksInProgress.remove(target); completedChecks.put(target, new LastCheckResult<>( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java index ae7b34f773468..d014e499f912e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java @@ -20,8 +20,8 @@ */ package org.apache.hadoop.hdfs.server.datanode.checker; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; import org.apache.hadoop.hdfs.server.datanode.checker.AbstractFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java index 963419688abbe..03cddb28c6ef1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode.erasurecode; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java index b2e64966a18b0..d9e2f60f76ff5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java @@ -87,7 +87,7 @@ public void reconstruct() throws IOException { // step3: calculate checksum checksumDataLen += checksumWithTargetOutput( - targetBuffer.array(), toReconstructLen); + getBufferArray(targetBuffer), toReconstructLen); updatePositionInBlock(toReconstructLen); requestedLen -= toReconstructLen; @@ -140,7 +140,7 @@ private long checksumWithTargetOutput(byte[] outputData, int toReconstructLen) // case-2) length of data bytes which is less than bytesPerCRC if (requestedLen <= toReconstructLen) { int remainingLen = Math.toIntExact(requestedLen); - outputData = Arrays.copyOf(targetBuffer.array(), remainingLen); + outputData = Arrays.copyOf(outputData, remainingLen); int partialLength = remainingLen % getChecksum().getBytesPerChecksum(); @@ -207,4 +207,19 @@ private void clearBuffers() { public long getChecksumDataLen() { return checksumDataLen; } -} + + /** + * Gets an array corresponding the buffer. + * @param buffer the input buffer. + * @return the array with content of the buffer. + */ + private static byte[] getBufferArray(ByteBuffer buffer) { + byte[] buff = new byte[buffer.remaining()]; + if (buffer.hasArray()) { + buff = buffer.array(); + } else { + buffer.slice().get(buff); + } + return buff; + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java index 98edf724a8e25..8624e353484b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode.erasurecode; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java index 762506cfdaefb..f955ad283e617 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode.erasurecode; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java index 177c62e017411..89ad510399641 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java @@ -237,16 +237,17 @@ StorageReport[] getStorageReports(String bpid) VolumeFailureSummary getVolumeFailureSummary(); /** - * Gets a list of references to the finalized blocks for the given block pool. + * Gets a sorted list of references to the finalized blocks for the given + * block pool. The list is sorted by blockID. *

* Callers of this function should call * {@link FsDatasetSpi#acquireDatasetLock} to avoid blocks' status being * changed during list iteration. *

* @return a list of references to the finalized blocks for the given block - * pool. + * pool. The list is sorted by blockID. */ - List getFinalizedBlocks(String bpid); + List getSortedFinalizedBlocks(String bpid); /** * Check whether the in-memory block record matches the block on the disk, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java index be978d75e9a34..68d1a15d5c3b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java @@ -77,6 +77,9 @@ public interface FsVolumeSpi /** Returns true if the volume is NOT backed by persistent storage. */ boolean isTransientStorage(); + /** Returns true if the volume is backed by RAM storage. */ + boolean isRAMStorage(); + /** * Reserve disk space for a block (RBW or Re-replicating) * so a writer does not run out of space before the block is full. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java index fc84c4d280cde..20df0e986b718 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java @@ -76,7 +76,7 @@ import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.Timer; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * A block pool slice represents a portion of a block pool stored on a volume. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/CacheStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/CacheStats.java index f79b7c7374c6e..476a31e2ab132 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/CacheStats.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/CacheStats.java @@ -22,7 +22,7 @@ import org.apache.hadoop.io.nativeio.NativeIO; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Keeps statistics for the memory cache. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java index 732dc2180eabf..b6a57fdeffa77 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java @@ -23,8 +23,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_CACHE_REVOCATION_POLLING_MS; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_CACHE_REVOCATION_POLLING_MS_DEFAULT; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.FileInputStream; import java.io.FileNotFoundException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index de898e93432f9..f5bfd92de0165 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -48,7 +48,7 @@ import javax.management.ObjectName; import javax.management.StandardMBean; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; @@ -121,9 +121,9 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Timer; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -171,7 +171,9 @@ public StorageReport[] getStorageReports(String bpid) volume.getDfsUsed(), volume.getAvailable(), volume.getBlockPoolUsed(bpid), - volume.getNonDfsUsed()); + volume.getNonDfsUsed(), + volume.getMount() + ); reports.add(sr); } catch (ClosedChannelException e) { continue; @@ -190,6 +192,10 @@ public FsVolumeImpl getVolume(final ExtendedBlock b) { } } + MountVolumeMap getMountVolumeMap() { + return volumes.getMountVolumeMap(); + } + @Override // FsDatasetSpi public Block getStoredBlock(String bpid, long blkid) throws IOException { @@ -365,7 +371,7 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b) RoundRobinVolumeChoosingPolicy.class, VolumeChoosingPolicy.class), conf); volumes = new FsVolumeList(volumeFailureInfos, datanode.getBlockScanner(), - blockChooserImpl); + blockChooserImpl, conf); asyncDiskService = new FsDatasetAsyncDiskService(datanode, this); asyncLazyPersistService = new RamDiskAsyncLazyPersistService(datanode, conf); deletingBlock = new HashMap>(); @@ -464,12 +470,27 @@ private void activateVolume( LOG.error(errorMsg); throw new IOException(errorMsg); } + // Check if there is same storage type on the mount. + // Only useful when same disk tiering is turned on. + FsVolumeImpl volumeImpl = (FsVolumeImpl) ref.getVolume(); + FsVolumeReference checkRef = volumes + .getMountVolumeMap() + .getVolumeRefByMountAndStorageType( + volumeImpl.getMount(), volumeImpl.getStorageType()); + if (checkRef != null) { + final String errorMsg = String.format( + "Storage type %s already exists on same mount: %s.", + volumeImpl.getStorageType(), volumeImpl.getMount()); + checkRef.close(); + LOG.error(errorMsg); + throw new IOException(errorMsg); + } volumeMap.mergeAll(replicaMap); storageMap.put(sd.getStorageUuid(), new DatanodeStorage(sd.getStorageUuid(), DatanodeStorage.State.NORMAL, storageType)); - asyncDiskService.addVolume((FsVolumeImpl) ref.getVolume()); + asyncDiskService.addVolume(volumeImpl); volumes.addVolume(ref); } } @@ -1957,28 +1978,32 @@ public Map getBlockReports(String bpid) { continue; } String volStorageID = b.getVolume().getStorageID(); - if (!builders.containsKey(volStorageID)) { - if (!missingVolumesReported.contains(volStorageID)) { - LOG.warn("Storage volume: " + volStorageID + " missing for the" - + " replica block: " + b + ". Probably being removed!"); - missingVolumesReported.add(volStorageID); - } - continue; - } switch(b.getState()) { case FINALIZED: case RBW: case RWR: - builders.get(volStorageID).add(b); break; case RUR: - ReplicaInfo orig = b.getOriginalReplica(); - builders.get(volStorageID).add(orig); + // use the original replica. + b = b.getOriginalReplica(); break; case TEMPORARY: - break; + continue; default: assert false : "Illegal ReplicaInfo state."; + continue; + } + BlockListAsLongs.Builder storageBuilder = builders.get(volStorageID); + // a storage in the process of failing will not be in the volumes list + // but will be in the replica map. + if (storageBuilder != null) { + storageBuilder.add(b); + } else { + if (!missingVolumesReported.contains(volStorageID)) { + LOG.warn("Storage volume: " + volStorageID + " missing for the" + + " replica block: " + b + ". Probably being removed!"); + missingVolumesReported.add(volStorageID); + } } } } @@ -1992,17 +2017,18 @@ public Map getBlockReports(String bpid) { } /** - * Gets a list of references to the finalized blocks for the given block pool. + * Gets a list of references to the finalized blocks for the given block pool, + * sorted by blockID. *

* Callers of this function should call * {@link FsDatasetSpi#acquireDatasetLock()} to avoid blocks' status being * changed during list iteration. *

* @return a list of references to the finalized blocks for the given block - * pool. + * pool. The list is sorted by blockID. */ @Override - public List getFinalizedBlocks(String bpid) { + public List getSortedFinalizedBlocks(String bpid) { try (AutoCloseableLock lock = datasetReadLock.acquire()) { final List finalized = new ArrayList( volumeMap.size(bpid)); @@ -2298,9 +2324,9 @@ private void cacheBlock(String bpid, long blockId) { ": volume was not an instance of FsVolumeImpl."); return; } - if (volume.isTransientStorage()) { - LOG.warn("Caching not supported on block with id " + blockId + - " since the volume is backed by RAM."); + if (volume.isRAMStorage()) { + LOG.warn("Caching not supported on block with id {} since the " + + "volume is backed by {} which is RAM.", blockId, volume); return; } success = true; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java index fbd02c76820cd..b5d2b95e219cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java @@ -34,7 +34,7 @@ import java.nio.file.Paths; import java.util.Arrays; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.protocol.Block; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java index 5843c7d6696b7..ccb76b178040b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java @@ -86,10 +86,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; /** * The underlying volume used to store replica. @@ -134,6 +134,9 @@ public class FsVolumeImpl implements FsVolumeSpi { private final FileIoProvider fileIoProvider; private final DataNodeVolumeMetrics metrics; private URI baseURI; + private boolean enableSameDiskTiering; + private final String mount; + private double reservedForArchive; /** * Per-volume worker pool that processes new blocks to cache. @@ -190,10 +193,22 @@ public class FsVolumeImpl implements FsVolumeSpi { } this.conf = conf; this.fileIoProvider = fileIoProvider; + this.enableSameDiskTiering = + conf.getBoolean(DFSConfigKeys.DFS_DATANODE_ALLOW_SAME_DISK_TIERING, + DFSConfigKeys.DFS_DATANODE_ALLOW_SAME_DISK_TIERING_DEFAULT); + if (enableSameDiskTiering && usage != null) { + this.mount = usage.getMount(); + } else { + mount = ""; + } + } + + String getMount() { + return mount; } protected ThreadPoolExecutor initializeCacheExecutor(File parent) { - if (storageType.isTransient()) { + if (storageType.isRAM()) { return null; } if (dataset.datanode == null) { @@ -407,11 +422,15 @@ long getBlockPoolUsed(String bpid) throws IOException { * Return either the configured capacity of the file system if configured; or * the capacity of the file system excluding space reserved for non-HDFS. * + * When same-disk-tiering is turned on, the reported capacity + * will take reservedForArchive value into consideration of. + * * @return the unreserved number of bytes left in this filesystem. May be * zero. */ @VisibleForTesting public long getCapacity() { + long capacity; if (configuredCapacity < 0L) { long remaining; if (cachedCapacity > 0L) { @@ -419,9 +438,18 @@ public long getCapacity() { } else { remaining = usage.getCapacity() - getReserved(); } - return Math.max(remaining, 0L); + capacity = Math.max(remaining, 0L); + } else { + capacity = configuredCapacity; + } + + if (enableSameDiskTiering && dataset.getMountVolumeMap() != null) { + double capacityRatio = dataset.getMountVolumeMap() + .getCapacityRatioByMountAndStorageType(mount, storageType); + capacity = (long) (capacity * capacityRatio); } - return configuredCapacity; + + return capacity; } /** @@ -452,7 +480,34 @@ public long getAvailable() throws IOException { } long getActualNonDfsUsed() throws IOException { - return usage.getUsed() - getDfsUsed(); + // DISK and ARCHIVAL on same disk + // should share the same amount of reserved capacity. + // When calculating actual non dfs used, + // exclude DFS used capacity by another volume. + if (enableSameDiskTiering && + (storageType == StorageType.DISK + || storageType == StorageType.ARCHIVE)) { + StorageType counterpartStorageType = storageType == StorageType.DISK + ? StorageType.ARCHIVE : StorageType.DISK; + FsVolumeReference counterpartRef = dataset + .getMountVolumeMap() + .getVolumeRefByMountAndStorageType(mount, counterpartStorageType); + if (counterpartRef != null) { + FsVolumeImpl counterpartVol = (FsVolumeImpl) counterpartRef.getVolume(); + long used = getDfUsed() - getDfsUsed() - counterpartVol.getDfsUsed(); + counterpartRef.close(); + return used; + } + } + return getDfUsed() - getDfsUsed(); + } + + /** + * This function is only used for Mock. + */ + @VisibleForTesting + public long getDfUsed() { + return usage.getUsed(); } private long getRemainingReserved() throws IOException { @@ -533,6 +588,11 @@ public boolean isTransientStorage() { return storageType.isTransient(); } + @Override + public boolean isRAMStorage() { + return storageType.isRAM(); + } + @VisibleForTesting public File getFinalizedDir(String bpid) throws IOException { return getBlockPoolSlice(bpid).getFinalizedDir(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplBuilder.java index 50ab97b87a7fc..5cdd45c70fcf2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplBuilder.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplBuilder.java @@ -19,7 +19,7 @@ import java.io.IOException; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.DF; import org.apache.hadoop.fs.StorageType; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index 049654b567dd5..2d6593df9bd04 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; @@ -62,9 +63,13 @@ class FsVolumeList { private final VolumeChoosingPolicy blockChooser; private final BlockScanner blockScanner; + private final boolean enableSameDiskTiering; + private final MountVolumeMap mountVolumeMap; + FsVolumeList(List initialVolumeFailureInfos, BlockScanner blockScanner, - VolumeChoosingPolicy blockChooser) { + VolumeChoosingPolicy blockChooser, + Configuration config) { this.blockChooser = blockChooser; this.blockScanner = blockScanner; this.checkDirsLock = new AutoCloseableLock(); @@ -73,6 +78,14 @@ class FsVolumeList { volumeFailureInfos.put(volumeFailureInfo.getFailedStorageLocation(), volumeFailureInfo); } + enableSameDiskTiering = config.getBoolean( + DFSConfigKeys.DFS_DATANODE_ALLOW_SAME_DISK_TIERING, + DFSConfigKeys.DFS_DATANODE_ALLOW_SAME_DISK_TIERING_DEFAULT); + mountVolumeMap = new MountVolumeMap(config); + } + + MountVolumeMap getMountVolumeMap() { + return mountVolumeMap; } /** @@ -291,6 +304,9 @@ public String toString() { void addVolume(FsVolumeReference ref) { FsVolumeImpl volume = (FsVolumeImpl) ref.getVolume(); volumes.add(volume); + if (isSameDiskTieringApplied(volume)) { + mountVolumeMap.addVolume(volume); + } if (blockScanner != null) { blockScanner.addVolumeScanner(ref); } else { @@ -311,6 +327,9 @@ void addVolume(FsVolumeReference ref) { */ private void removeVolume(FsVolumeImpl target) { if (volumes.remove(target)) { + if (isSameDiskTieringApplied(target)) { + mountVolumeMap.removeVolume(target); + } if (blockScanner != null) { blockScanner.removeVolumeScanner(target); } @@ -331,6 +350,15 @@ private void removeVolume(FsVolumeImpl target) { } } + /** + * Check if same disk tiering is applied to the volume. + */ + private boolean isSameDiskTieringApplied(FsVolumeImpl target) { + return enableSameDiskTiering && + (target.getStorageType() == StorageType.DISK + || target.getStorageType() == StorageType.ARCHIVE); + } + /** * Dynamically remove volume in the list. * @param storageLocation {@link StorageLocation} of the volume to be removed. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MappableBlockLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MappableBlockLoader.java index 02bea81f70d20..96d88345e6b9d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MappableBlockLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MappableBlockLoader.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.io.IOUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MountVolumeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MountVolumeInfo.java new file mode 100644 index 0000000000000..660cae26e40a3 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MountVolumeInfo.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; + +import java.nio.channels.ClosedChannelException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * MountVolumeInfo is a wrapper of + * detailed volume information for MountVolumeMap. + */ +@InterfaceAudience.Private +class MountVolumeInfo { + private final ConcurrentMap + storageTypeVolumeMap; + private double reservedForArchiveDefault; + + MountVolumeInfo(Configuration conf) { + storageTypeVolumeMap = new ConcurrentHashMap<>(); + reservedForArchiveDefault = conf.getDouble( + DFSConfigKeys.DFS_DATANODE_RESERVE_FOR_ARCHIVE_DEFAULT_PERCENTAGE, + DFSConfigKeys + .DFS_DATANODE_RESERVE_FOR_ARCHIVE_DEFAULT_PERCENTAGE_DEFAULT); + if (reservedForArchiveDefault > 1) { + FsDatasetImpl.LOG.warn("Value of reserve-for-archival is > 100%." + + " Setting it to 100%."); + reservedForArchiveDefault = 1; + } + if (reservedForArchiveDefault < 0) { + FsDatasetImpl.LOG.warn("Value of reserve-for-archival is < 0." + + " Setting it to 0.0"); + reservedForArchiveDefault = 0; + } + } + + FsVolumeReference getVolumeRef(StorageType storageType) { + try { + FsVolumeImpl volumeImpl = storageTypeVolumeMap + .getOrDefault(storageType, null); + if (volumeImpl != null) { + return volumeImpl.obtainReference(); + } + } catch (ClosedChannelException e) { + FsDatasetImpl.LOG.warn("Volume closed when getting volume" + + " by storage type: " + storageType); + } + return null; + } + + /** + * Return configured capacity ratio. + * If the volume is the only one on the mount, + * return 1 to avoid unnecessary allocation. + * + * TODO: We should support customized capacity ratio for volumes. + */ + double getCapacityRatio(StorageType storageType) { + if (storageTypeVolumeMap.containsKey(storageType) + && storageTypeVolumeMap.size() > 1) { + if (storageType == StorageType.ARCHIVE) { + return reservedForArchiveDefault; + } else if (storageType == StorageType.DISK) { + return 1 - reservedForArchiveDefault; + } + } + return 1; + } + + /** + * Add a volume to the mapping. + * If there is already storage type exists on same mount, skip this volume. + */ + boolean addVolume(FsVolumeImpl volume) { + if (storageTypeVolumeMap.containsKey(volume.getStorageType())) { + FsDatasetImpl.LOG.error("Found storage type already exist." + + " Skipping for now. Please check disk configuration"); + return false; + } + storageTypeVolumeMap.put(volume.getStorageType(), volume); + return true; + } + + + void removeVolume(FsVolumeImpl target) { + storageTypeVolumeMap.remove(target.getStorageType()); + } + + int size() { + return storageTypeVolumeMap.size(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MountVolumeMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MountVolumeMap.java new file mode 100644 index 0000000000000..6fe4d3a690a5d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MountVolumeMap.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * MountVolumeMap contains information of the relationship + * between underlying filesystem mount and datanode volumes. + * + * This is useful when configuring block tiering on same disk mount + * (HDFS-15548). For now, + * we don't configure multiple volumes with same storage type on one mount. + */ +@InterfaceAudience.Private +class MountVolumeMap { + private final ConcurrentMap + mountVolumeMapping; + private final Configuration conf; + + MountVolumeMap(Configuration conf) { + mountVolumeMapping = new ConcurrentHashMap<>(); + this.conf = conf; + } + + FsVolumeReference getVolumeRefByMountAndStorageType(String mount, + StorageType storageType) { + if (mountVolumeMapping.containsKey(mount)) { + return mountVolumeMapping + .get(mount).getVolumeRef(storageType); + } + return null; + } + + /** + * Return capacity ratio. + * If not exists, return 1 to use full capacity. + */ + double getCapacityRatioByMountAndStorageType(String mount, + StorageType storageType) { + if (mountVolumeMapping.containsKey(mount)) { + return mountVolumeMapping.get(mount).getCapacityRatio(storageType); + } + return 1; + } + + void addVolume(FsVolumeImpl volume) { + String mount = volume.getMount(); + if (!mount.isEmpty()) { + MountVolumeInfo info; + if (mountVolumeMapping.containsKey(mount)) { + info = mountVolumeMapping.get(mount); + } else { + info = new MountVolumeInfo(conf); + mountVolumeMapping.put(mount, info); + } + info.addVolume(volume); + } + } + + void removeVolume(FsVolumeImpl target) { + String mount = target.getMount(); + if (!mount.isEmpty()) { + MountVolumeInfo info = mountVolumeMapping.get(mount); + info.removeVolume(target); + if (info.size() == 0) { + mountVolumeMapping.remove(mount); + } + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/NativePmemMappableBlockLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/NativePmemMappableBlockLoader.java index 55e5decd0832c..ec024cda9ab02 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/NativePmemMappableBlockLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/NativePmemMappableBlockLoader.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.io.IOUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/PmemVolumeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/PmemVolumeManager.java index 51b768198b7a2..a85c577745af4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/PmemVolumeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/PmemVolumeManager.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.io.filefilter.TrueFileFilter; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java index b4d2c640cd804..7e077181707af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java @@ -65,7 +65,7 @@ import org.codehaus.jackson.map.ObjectReader; import org.codehaus.jackson.map.ObjectWriter; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * This class is used to create provided volumes. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java index b940736ccfd28..31e9ebe0b8c2e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; -import com.google.common.collect.TreeMultimap; +import org.apache.hadoop.thirdparty.com.google.common.collect.TreeMultimap; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.util.Time; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java index 07e520117f617..f7b12ff179941 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java index a8a6c85762d62..e431bde9f15eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hdfs.server.datanode.metrics; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.server.datanode.DataNode; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java index 401fc8bd712d3..a30baa1d23bea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hdfs.server.datanode.metrics; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.slf4j.Logger; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/HostRestrictingAuthorizationFilterHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/HostRestrictingAuthorizationFilterHandler.java index 584e12bdc9b02..798def0c716b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/HostRestrictingAuthorizationFilterHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/HostRestrictingAuthorizationFilterHandler.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode.web; -import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandler.Sharable; import io.netty.channel.ChannelHandlerContext; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java index 366f47f29631d..d8fee43c6840c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java @@ -23,9 +23,9 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.cache.Cache; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; import java.io.ByteArrayInputStream; import java.io.DataInputStream; @@ -72,9 +72,12 @@ UserGroupInformation ugi() throws IOException { UserGroupInformation ugi; try { - if (UserGroupInformation.isSecurityEnabled()) { - final Token token = params.delegationToken(); + final Token token = params.delegationToken(); + // Create nonTokenUGI when token is null regardless of security. + // This makes it possible to access the data stored in secure DataNode + // through insecure Namenode. + if (UserGroupInformation.isSecurityEnabled() && token != null) { ugi = ugiCache.get(buildTokenCacheKey(token), new Callable() { @Override @@ -134,7 +137,8 @@ private String buildNonTokenCacheKey(String doAsUserFromQuery, return key; } - private UserGroupInformation nonTokenUGI(String usernameFromQuery, + @VisibleForTesting + UserGroupInformation nonTokenUGI(String usernameFromQuery, String doAsUserFromQuery, String remoteUser) throws IOException { UserGroupInformation ugi = UserGroupInformation diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java index b56b0d09ac28b..02ec25c13c874 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode.web.webhdfs; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import com.sun.jersey.api.ParamException; import com.sun.jersey.api.container.ContainerException; import io.netty.buffer.Unpooled; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java index 2b3a39358de6e..c4588fc65744e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java @@ -123,6 +123,9 @@ boolean noredirect() { Token delegationToken() throws IOException { String delegation = param(DelegationParam.NAME); + if (delegation == null) { + return null; + } final Token token = new Token(); token.decodeFromUrlString(delegation); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java index ff68a7ee7e4d0..2f901da165ff4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode.web.webhdfs; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import io.netty.buffer.Unpooled; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandlerContext; @@ -335,8 +335,8 @@ private static void writeContinueHeader(ChannelHandlerContext ctx) { } private void injectToken() throws IOException { - if (UserGroupInformation.isSecurityEnabled()) { - Token token = params.delegationToken(); + Token token = params.delegationToken(); + if (UserGroupInformation.isSecurityEnabled() && token != null) { token.setKind(HDFS_DELEGATION_KIND); ugi.addToken(token); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java index 8cd4e1482d83b..4e7b38c2f3406 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer.command; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.codec.digest.DigestUtils; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java index 4f8e373d9c2ce..6845c572ef6b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java @@ -20,9 +20,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java index be4a3e3aa217e..7eac88113866d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer.command; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.codec.digest.DigestUtils; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java index f7c84e16f7b81..e36628edf0eb2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer.command; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java index ebcbb4c2fa9f9..e9f9f33e71535 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.server.diskbalancer.command; -import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Throwables; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java index a8adcbd5621bb..520e80f3974c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer.command; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java index 4f75aff1a518c..ad5a3c2090edf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java @@ -33,8 +33,8 @@ import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolumeSet; import org.apache.hadoop.hdfs.tools.DiskBalancerCLI; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * Executes the report command. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/DBNameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/DBNameNodeConnector.java index 2d8ba8a0a0724..b7bb3f02dce9d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/DBNameNodeConnector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/DBNameNodeConnector.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer.connectors; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java index 838511ba82bf8..268c055a354ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java index 1307983c80927..0e405ff7bd3c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java index 6cf244be9442d..fce858aaca01b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer.datamodel; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import java.util.HashMap; import java.util.Map; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolumeSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolumeSet.java index 5a4eb6d68554d..bcce012ff84b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolumeSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolumeSet.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java index 3f9734537a52b..0ed56afb39a5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer.planner; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hdfs.server.diskbalancer.datamodel diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java index 44039eaa0f36e..72df5abe6bcaa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import java.io.IOException; import java.util.LinkedList; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java index 06c7cc5a5acf7..cae6b68793580 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hdfs.server.mover; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.commons.cli.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -134,9 +134,17 @@ private List getTargetStorages(StorageType t) { final int maxNoMoveInterval = conf.getInt( DFSConfigKeys.DFS_MOVER_MAX_NO_MOVE_INTERVAL_KEY, DFSConfigKeys.DFS_MOVER_MAX_NO_MOVE_INTERVAL_DEFAULT); - this.retryMaxAttempts = conf.getInt( + final int maxAttempts = conf.getInt( DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY, DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_DEFAULT); + if (maxAttempts >= 0) { + this.retryMaxAttempts = maxAttempts; + } else { + LOG.warn(DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY + " is " + + "configured with a negative value, using default value of " + + DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_DEFAULT); + this.retryMaxAttempts = DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_DEFAULT; + } this.retryCount = retryCount; this.dispatcher = new Dispatcher(nnc, Collections. emptySet(), Collections. emptySet(), movedWinWidth, moverThreads, 0, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java index e9e3e598fc12a..a610aee48496c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java @@ -26,7 +26,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hdfs.util.LongBitFormat; -import com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; /** * Class to pack an AclEntry into an integer.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java index 6d546f81c4f99..d18079ee898b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java @@ -24,8 +24,8 @@ import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.hdfs.util.ReferenceCountMap.ReferenceCounter; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; /** * Feature that represents the ACLs of the inode. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java index fa268c72a1eb2..806cdc6d6940b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java @@ -20,9 +20,9 @@ import java.util.Collections; import java.util.List; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.permission.AclEntry; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java index 4402e263a2108..031929ce67c79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java @@ -28,10 +28,10 @@ import java.util.Iterator; import java.util.List; -import com.google.common.collect.ComparisonChain; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Ordering; +import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Ordering; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.permission.AclEntry; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java index 9f5f29e371432..bbe607670f71a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java @@ -30,8 +30,8 @@ import org.apache.hadoop.hdfs.server.common.Storage.StorageState; import org.apache.hadoop.util.StringUtils; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * Extension of FSImage for the backup node. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java index 2fb1933c421cc..dab227fcc763c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java @@ -51,7 +51,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.protobuf.BlockingService; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java index 88d71143ae269..68ab12c9eb4e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java @@ -90,10 +90,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Lists; -import com.google.common.collect.Multimap; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultimap; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Multimap; /** * The Cache Manager handles caching on DataNodes. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java index 20b1d25434a1b..dda4789b74e49 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java @@ -32,7 +32,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.IntrusiveCollection; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * A CachePool describes a set of cache resources being managed by the NameNode. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java index 186bc3d727639..4df170d771601 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java @@ -24,7 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import java.util.concurrent.TimeUnit; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java index 18cc7afbe9d5c..5c753070a459e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; -import com.google.common.collect.ComparisonChain; +import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; /** * A unique signature intended to identify checkpoint transactions. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java index ab07efa81c77a..0557580404a7f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java @@ -28,7 +28,7 @@ import java.net.URL; import java.util.List; -import com.google.common.math.LongMath; +import org.apache.hadoop.thirdparty.com.google.common.math.LongMath; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -43,7 +43,7 @@ import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.util.Daemon; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * The Checkpointer is responsible for supporting periodic checkpoints diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java index 0263f2a347faa..7a5963a6c57cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.permission.FsAction; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java index 81d285a03626b..d17fd06bc882c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java @@ -21,7 +21,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java index 26c5e9049f74d..8c3c0fd933a43 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java @@ -45,9 +45,9 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticationException; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Throwables; /** * An implementation of the abstract class {@link EditLogInputStream}, which diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java index 4dbe2720ddaa0..4919ea44e0f72 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hdfs.protocol.LayoutFlags; import org.apache.hadoop.io.IOUtils; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * An implementation of the abstract class {@link EditLogOutputStream}, which diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java index be68f6d609008..affba028c08cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java @@ -32,7 +32,7 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.IOUtils; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * A double-buffer for edits. New edits are written into the first buffer diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java index 938eacd7fd648..33954a2cc3993 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java @@ -19,7 +19,7 @@ import java.io.IOException; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Used to inject certain faults for testing. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java index 48e405ba35432..2f5fde8e94015 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java @@ -29,10 +29,10 @@ import java.util.NavigableMap; import java.util.TreeMap; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index d9f7e9afdc70d..73f5596eabf36 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java index 31dc51a3c24b1..6ced588e96882 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.permission.AclEntry; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java index 919e853f4590f..2586a257beff3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion.Feature; import org.apache.hadoop.ipc.RetriableException; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Helper class to perform append operation. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java index 8e9606dcf61fe..cfadbeb5eb4aa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hdfs.util.EnumCounters; import org.apache.hadoop.security.AccessControlException; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import java.io.FileNotFoundException; import java.io.IOException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java index ebd7b6020abab..843f1a5464da3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.fs.permission.FsAction; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java index 6c7b1fae50dec..516d59415401c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java @@ -52,8 +52,8 @@ import org.apache.hadoop.hdfs.server.namenode.ReencryptionUpdater.FileEdekInfo; import org.apache.hadoop.security.SecurityUtil; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.util.Time; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index 011c72ea49c6f..11981b27183d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.fs.FileStatus; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java index 95e889888bb7f..da324fb46738a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; import org.apache.hadoop.fs.permission.FsCreateModes; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.ParentNotDirectoryException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java index 602f9962942be..8f6fcdc1d1f02 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.InvalidPathException; @@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager; import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.util.ChunkedArrayList; import org.apache.hadoop.util.Time; @@ -193,7 +194,7 @@ static INodesInPath unprotectedRenameTo(FSDirectory fsd, } validateNestSnapshot(fsd, src, dstParent.asDirectory(), snapshottableDirs); - + checkUnderSameSnapshottableRoot(fsd, srcIIP, dstIIP); fsd.ezManager.checkMoveValidity(srcIIP, dstIIP); // Ensure dst has quota to accommodate rename verifyFsLimitsForRename(fsd, srcIIP, dstIIP); @@ -262,6 +263,11 @@ static RenameResult renameTo(FSDirectory fsd, FSPermissionChecker pc, throws IOException { final INodesInPath srcIIP = fsd.resolvePath(pc, src, DirOp.WRITE_LINK); final INodesInPath dstIIP = fsd.resolvePath(pc, dst, DirOp.CREATE_LINK); + + if(fsd.isNonEmptyDirectory(srcIIP)) { + DFSUtil.checkProtectedDescendants(fsd, srcIIP); + } + if (fsd.isPermissionEnabled()) { boolean renameToTrash = false; if (null != options && @@ -407,6 +413,7 @@ static RenameResult unprotectedRenameTo(FSDirectory fsd, validateNestSnapshot(fsd, src, dstParent.asDirectory(), srcSnapshottableDirs); + checkUnderSameSnapshottableRoot(fsd, srcIIP, dstIIP); // Ensure dst has quota to accommodate rename verifyFsLimitsForRename(fsd, srcIIP, dstIIP); @@ -650,7 +657,12 @@ private static class RenameOperation { // snapshot is taken on the dst tree, changes will be recorded in the // latest snapshot of the src tree. if (isSrcInSnapshot) { - srcChild.recordModification(srcLatestSnapshotId); + if (srcChild.isFile()) { + INodeFile file = srcChild.asFile(); + file.recordModification(srcLatestSnapshotId, true); + } else { + srcChild.recordModification(srcLatestSnapshotId); + } } // check srcChild for reference @@ -816,6 +828,29 @@ void updateQuotasInSourceTree(BlockStoragePolicySuite bsps) { } } + private static void checkUnderSameSnapshottableRoot( + FSDirectory fsd, INodesInPath srcIIP, INodesInPath dstIIP) + throws IOException { + // Ensure rename out of a snapshottable root is not permitted if ordered + // snapshot deletion feature is enabled + SnapshotManager snapshotManager = fsd.getFSNamesystem(). + getSnapshotManager(); + if (snapshotManager.isSnapshotDeletionOrdered() && fsd.getFSNamesystem() + .isSnapshotTrashRootEnabled()) { + INodeDirectory srcRoot = snapshotManager. + getSnapshottableAncestorDir(srcIIP); + INodeDirectory dstRoot = snapshotManager. + getSnapshottableAncestorDir(dstIIP); + // Ensure snapshoottable root for both src and dest are same. + if (srcRoot != dstRoot) { + String errMsg = "Source " + srcIIP.getPath() + + " and dest " + dstIIP.getPath() + " are not under " + + "the same snapshot root."; + throw new SnapshotException(errMsg); + } + } + } + private static RenameResult createRenameResult(FSDirectory fsd, INodesInPath dst, boolean filesDeleted, BlocksMapUpdateInfo collectedBlocks) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSatisfyStoragePolicyOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSatisfyStoragePolicyOp.java index 3f873d7eea5ff..4057bbd211c0a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSatisfyStoragePolicyOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSatisfyStoragePolicyOp.java @@ -33,7 +33,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp; import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfyManager; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * Helper class to perform storage policy satisfier related operations. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java index c854f83142e5f..d45c0c30d4907 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; @@ -154,6 +155,22 @@ static SnapshottableDirectoryStatus[] getSnapshottableDirListing( } } + static SnapshotStatus[] getSnapshotListing( + FSDirectory fsd, FSPermissionChecker pc, SnapshotManager snapshotManager, + String path) + throws IOException { + fsd.readLock(); + try { + INodesInPath iip = fsd.getINodesInPath(path, DirOp.READ); + if (fsd.isPermissionEnabled()) { + fsd.checkPathAccess(pc, iip, FsAction.READ); + } + return snapshotManager.getSnapshotListing(iip); + } finally { + fsd.readUnlock(); + } + } + static SnapshotDiffReport getSnapshotDiffReport(FSDirectory fsd, FSPermissionChecker pc, SnapshotManager snapshotManager, String path, String fromSnapshot, String toSnapshot) throws IOException { @@ -249,12 +266,22 @@ static INode.BlocksMapUpdateInfo deleteSnapshot( fsd.checkOwner(pc, iip); } + // time of snapshot deletion + final long now = Time.now(); + final INode.BlocksMapUpdateInfo collectedBlocks = deleteSnapshot( + fsd, snapshotManager, iip, snapshotName, now, snapshotRoot, + logRetryCache); + return collectedBlocks; + } + + static INode.BlocksMapUpdateInfo deleteSnapshot( + FSDirectory fsd, SnapshotManager snapshotManager, INodesInPath iip, + String snapshotName, long now, String snapshotRoot, boolean logRetryCache) + throws IOException { INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo(); ChunkedArrayList removedINodes = new ChunkedArrayList<>(); INode.ReclaimContext context = new INode.ReclaimContext( fsd.getBlockStoragePolicySuite(), collectedBlocks, removedINodes, null); - // time of snapshot deletion - final long now = Time.now(); fsd.writeLock(); try { snapshotManager.deleteSnapshot(iip, snapshotName, context, now); @@ -268,7 +295,6 @@ static INode.BlocksMapUpdateInfo deleteSnapshot( removedINodes.clear(); fsd.getEditLog().logDeleteSnapshot(snapshotRoot, snapshotName, logRetryCache, now); - return collectedBlocks; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java index 7f07dd1feda1b..dfacc491eae53 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java index ee50ee92a8320..4104930ca3659 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.RecoverLeaseOp; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Helper class to perform truncate operation. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java index 99a80b537c208..0d9c6aeeb9c45 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.hdfs.AddBlockFlag; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java index ff82610f545bb..ef83fe982b24c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.fs.FileStatus; @@ -41,12 +41,13 @@ import java.util.List; import java.util.ListIterator; -import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_FILE_ENCRYPTION_INFO; +import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_SNAPSHOT_DELETED; +import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE; -class FSDirXAttrOp { +public class FSDirXAttrOp { private static final XAttr KEYID_XATTR = XAttrHelper.buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, null); private static final XAttr UNREADABLE_BY_SUPERUSER_XATTR = @@ -265,7 +266,7 @@ static List filterINodeXAttrs( return newXAttrs; } - static INode unprotectedSetXAttrs( + public static INode unprotectedSetXAttrs( FSDirectory fsd, final INodesInPath iip, final List xAttrs, final EnumSet flag) throws IOException { @@ -326,6 +327,12 @@ static INode unprotectedSetXAttrs( throw new IOException("Can only set '" + SECURITY_XATTR_UNREADABLE_BY_SUPERUSER + "' on a file."); } + + if (xaName.equals(XATTR_SNAPSHOT_DELETED) && !(inode.isDirectory() && + inode.getParent().isSnapshottable())) { + throw new IOException("Can only set '" + + XATTR_SNAPSHOT_DELETED + "' on a snapshot root."); + } } XAttrStorage.updateINodeXAttrs(inode, newXAttrs, iip.getLatestSnapshotId()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index fb5c9df8debdf..0e15921ba4953 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -20,9 +20,9 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.util.StringUtils; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.HadoopIllegalArgumentException; @@ -508,6 +508,14 @@ FSNamesystem getFSNamesystem() { return namesystem; } + /** + * Indicates whether the image loading is complete or not. + * @return true if image loading is complete, false otherwise + */ + public boolean isImageLoaded() { + return namesystem.isImageLoaded(); + } + /** * Parse configuration setting dfs.namenode.protected.directories to * retrieve the set of protected directories. @@ -610,6 +618,7 @@ boolean isXattrsEnabled() { return xattrsEnabled; } int getXattrMaxSize() { return xattrMaxSize; } + boolean isAccessTimeSupported() { return accessTimePrecision > 0; } @@ -1209,7 +1218,8 @@ static void verifyQuota(INodesInPath iip, int pos, QuotaCounts deltas, // check existing components in the path for(int i = (pos > iip.length() ? iip.length(): pos) - 1; i >= 0; i--) { - if (commonAncestor == iip.getINode(i)) { + if (commonAncestor == iip.getINode(i) + && !commonAncestor.isInLatestSnapshot(iip.getLatestSnapshotId())) { // Stop checking for quota when common ancestor is reached return; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index be8c684f01508..2ef3a028acd2a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -109,9 +109,9 @@ import org.apache.hadoop.ipc.Server; import org.apache.hadoop.security.token.delegation.DelegationKey; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java index 2b47398f40c98..e73dfa7797df8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java @@ -33,8 +33,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.util.ExitUtil; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; class FSEditLogAsync extends FSEditLog implements Runnable { static final Logger LOG = LoggerFactory.getLogger(FSEditLog.class); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index e3694ba4f4ca8..2ac0eb15b949f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -116,9 +116,9 @@ import org.apache.hadoop.util.ChunkedArrayList; import org.apache.hadoop.util.Timer; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import static org.apache.hadoop.log.LogThrottlingHelper.LogAction; @@ -1238,7 +1238,7 @@ private void incrOpCount(FSEditLogOpCodes opCode, holder = new Holder(1); opCounts.put(opCode, holder); } else { - holder.held++; + holder.held = holder.held + 1; } counter.increment(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java index 963628f9ac4e3..feff8b48f7e21 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java @@ -137,11 +137,11 @@ import org.xml.sax.SAXException; import org.xml.sax.helpers.AttributesImpl; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * Helper classes for reading the ops from an InputStream. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 9b95f14bddc28..ccc693492973a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -75,9 +75,9 @@ import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.Time; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * FSImage handles checkpointing and logging of the namespace edits. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java index da067d15b6db5..dcc60bf5e7152 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java @@ -75,8 +75,8 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.util.StringUtils; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Contains inner classes for reading or writing the on-disk format for diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index 6212e65e01db6..0a69c99cab810 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -28,8 +28,9 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,8 +72,8 @@ import org.apache.hadoop.hdfs.util.EnumCounters; import org.apache.hadoop.hdfs.util.ReadOnlyList; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hadoop.thirdparty.protobuf.ByteString; @InterfaceAudience.Private @@ -204,15 +205,20 @@ public static void updateBlocksMap(INodeFile file, BlockManager bm) { private final FSDirectory dir; private final FSNamesystem fsn; private final FSImageFormatProtobuf.Loader parent; - private ReentrantLock cacheNameMapLock; - private ReentrantLock blockMapLock; + + // Update blocks map by single thread asynchronously + private ExecutorService blocksMapUpdateExecutor; + // update name cache by single thread asynchronously. + private ExecutorService nameCacheUpdateExecutor; Loader(FSNamesystem fsn, final FSImageFormatProtobuf.Loader parent) { this.fsn = fsn; this.dir = fsn.dir; this.parent = parent; - cacheNameMapLock = new ReentrantLock(true); - blockMapLock = new ReentrantLock(true); + // Note: these executors must be SingleThreadExecutor, as they + // are used to modify structures which are not thread safe. + blocksMapUpdateExecutor = Executors.newSingleThreadExecutor(); + nameCacheUpdateExecutor = Executors.newSingleThreadExecutor(); } void loadINodeDirectorySectionInParallel(ExecutorService service, @@ -263,7 +269,6 @@ void loadINodeDirectorySectionInParallel(ExecutorService service, void loadINodeDirectorySection(InputStream in) throws IOException { final List refList = parent.getLoaderContext() .getRefList(); - ArrayList inodeList = new ArrayList<>(); while (true) { INodeDirectorySection.DirEntry e = INodeDirectorySection.DirEntry .parseDelimitedFrom(in); @@ -274,15 +279,7 @@ void loadINodeDirectorySection(InputStream in) throws IOException { INodeDirectory p = dir.getInode(e.getParent()).asDirectory(); for (long id : e.getChildrenList()) { INode child = dir.getInode(id); - if (addToParent(p, child)) { - if (child.isFile()) { - inodeList.add(child); - } - if (inodeList.size() >= DIRECTORY_ENTRY_BATCH_SIZE) { - addToCacheAndBlockMap(inodeList); - inodeList.clear(); - } - } else { + if (!addToParent(p, child)) { LOG.warn("Failed to add the inode {} to the directory {}", child.getId(), p.getId()); } @@ -290,40 +287,79 @@ void loadINodeDirectorySection(InputStream in) throws IOException { for (int refId : e.getRefChildrenList()) { INodeReference ref = refList.get(refId); - if (addToParent(p, ref)) { - if (ref.isFile()) { - inodeList.add(ref); - } - if (inodeList.size() >= DIRECTORY_ENTRY_BATCH_SIZE) { - addToCacheAndBlockMap(inodeList); - inodeList.clear(); - } - } else { + if (!addToParent(p, ref)) { LOG.warn("Failed to add the inode reference {} to the directory {}", ref.getId(), p.getId()); } } } - addToCacheAndBlockMap(inodeList); } - private void addToCacheAndBlockMap(ArrayList inodeList) { - try { - cacheNameMapLock.lock(); - for (INode i : inodeList) { - dir.cacheName(i); - } - } finally { - cacheNameMapLock.unlock(); + private void fillUpInodeList(ArrayList inodeList, INode inode) { + if (inode.isFile()) { + inodeList.add(inode); } + if (inodeList.size() >= DIRECTORY_ENTRY_BATCH_SIZE) { + addToCacheAndBlockMap(inodeList); + inodeList.clear(); + } + } - try { - blockMapLock.lock(); - for (INode i : inodeList) { - updateBlocksMap(i.asFile(), fsn.getBlockManager()); + private void addToCacheAndBlockMap(final ArrayList inodeList) { + final ArrayList inodes = new ArrayList<>(inodeList); + nameCacheUpdateExecutor.submit( + new Runnable() { + @Override + public void run() { + addToCacheInternal(inodes); + } + }); + blocksMapUpdateExecutor.submit( + new Runnable() { + @Override + public void run() { + updateBlockMapInternal(inodes); + } + }); + } + + // update name cache with non-thread safe + private void addToCacheInternal(ArrayList inodeList) { + for (INode i : inodeList) { + dir.cacheName(i); + } + } + + // update blocks map with non-thread safe + private void updateBlockMapInternal(ArrayList inodeList) { + for (INode i : inodeList) { + updateBlocksMap(i.asFile(), fsn.getBlockManager()); + } + } + + void waitBlocksMapAndNameCacheUpdateFinished() throws IOException { + long start = System.currentTimeMillis(); + waitExecutorTerminated(blocksMapUpdateExecutor); + waitExecutorTerminated(nameCacheUpdateExecutor); + LOG.info("Completed update blocks map and name cache, total waiting " + + "duration {}ms.", (System.currentTimeMillis() - start)); + } + + private void waitExecutorTerminated(ExecutorService executorService) + throws IOException { + executorService.shutdown(); + long start = System.currentTimeMillis(); + while (!executorService.isTerminated()) { + try { + executorService.awaitTermination(1, TimeUnit.SECONDS); + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting to executor service terminated duration {}ms.", + (System.currentTimeMillis() - start)); + } + } catch (InterruptedException e) { + LOG.error("Interrupted waiting for executor terminated.", e); + throw new IOException(e); } - } finally { - blockMapLock.unlock(); } } @@ -340,6 +376,7 @@ private int loadINodesInSection(InputStream in, Counter counter) // As the input stream is a LimitInputStream, the reading will stop when // EOF is encountered at the end of the stream. int cntr = 0; + ArrayList inodeList = new ArrayList<>(); while (true) { INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in); if (p == null) { @@ -354,12 +391,16 @@ private int loadINodesInSection(InputStream in, Counter counter) synchronized(this) { dir.addToInodeMap(n); } + fillUpInodeList(inodeList, n); } cntr++; if (counter != null) { counter.increment(); } } + if (inodeList.size() > 0){ + addToCacheAndBlockMap(inodeList); + } return cntr; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java index be21d1f80f9e9..e3a64537adc9d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java @@ -76,8 +76,8 @@ import org.apache.hadoop.util.LimitInputStream; import org.apache.hadoop.util.Time; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; /** @@ -447,6 +447,7 @@ public int compare(FileSummary.Section s1, FileSummary.Section s2) { } else { inodeLoader.loadINodeDirectorySection(in); } + inodeLoader.waitBlocksMapAndNameCacheUpdateFinished(); break; case FILES_UNDERCONSTRUCTION: inodeLoader.loadFilesUnderConstructionSection(in); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java index 706b2bda06299..3d75cebf729d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java @@ -53,7 +53,7 @@ import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Static utility functions for serializing various pieces of data in the correct diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java index b04007513c3fd..9f71f69902c86 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java @@ -35,8 +35,8 @@ import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; class FSImageTransactionalStorageInspector extends FSImageStorageInspector { public static final Logger LOG = LoggerFactory.getLogger( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index fe39b071e207c..4d740718a11f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -99,6 +99,7 @@ import org.apache.commons.text.CaseUtils; import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY; import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*; import static org.apache.hadoop.ha.HAServiceProtocol.HAServiceState.ACTIVE; @@ -107,7 +108,8 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; +import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotDeletionGc; import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.hdfs.protocol.BatchedDirectoryListing; import org.apache.hadoop.hdfs.protocol.HdfsPartialListing; @@ -337,12 +339,12 @@ import org.apache.log4j.AsyncAppender; import org.eclipse.jetty.util.ajax.JSON; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Charsets; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.LoggerFactory; /** @@ -380,6 +382,15 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, public static final org.slf4j.Logger LOG = LoggerFactory .getLogger(FSNamesystem.class.getName()); + + // The following are private configurations + public static final String DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED = + "dfs.namenode.snapshot.trashroot.enabled"; + public static final boolean DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED_DEFAULT + = false; + private static final FsPermission SHARED_TRASH_PERMISSION = + new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, true); + private final MetricsRegistry registry = new MetricsRegistry("FSNamesystem"); @Metric final MutableRatesWithAggregation detailedLockHoldTimeMetrics = registry.newRatesWithAggregation("detailedLockHoldTimeMetrics"); @@ -460,6 +471,7 @@ private void logAuditEvent(boolean succeeded, private final UserGroupInformation fsOwner; private final String supergroup; private final boolean standbyShouldCheckpoint; + private final boolean isSnapshotTrashRootEnabled; private final int snapshotDiffReportLimit; private final int blockDeletionIncrement; @@ -492,6 +504,7 @@ private void logAuditEvent(boolean succeeded, FSDirectory dir; private BlockManager blockManager; private final SnapshotManager snapshotManager; + private final SnapshotDeletionGc snapshotDeletionGc; private final CacheManager cacheManager; private final DatanodeStatistics datanodeStatistics; @@ -873,6 +886,9 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { // Get the checksum type from config String checksumTypeStr = conf.get(DFS_CHECKSUM_TYPE_KEY, DFS_CHECKSUM_TYPE_DEFAULT); + this.isSnapshotTrashRootEnabled = conf.getBoolean( + DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED, + DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED_DEFAULT); DataChecksum.Type checksumType; try { checksumType = DataChecksum.Type.valueOf(checksumTypeStr); @@ -899,7 +915,8 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { conf.getTrimmed( CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, ""), - blockManager.getStoragePolicySuite().getDefaultPolicy().getId()); + blockManager.getStoragePolicySuite().getDefaultPolicy().getId(), + isSnapshotTrashRootEnabled); this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY, DFS_NAMENODE_MAX_OBJECTS_DEFAULT); @@ -973,6 +990,9 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { this.dtSecretManager = createDelegationTokenSecretManager(conf); this.dir = new FSDirectory(this, conf); this.snapshotManager = new SnapshotManager(conf, dir); + this.snapshotDeletionGc = snapshotManager.isSnapshotDeletionOrdered()? + new SnapshotDeletionGc(this, conf): null; + this.cacheManager = new CacheManager(this, conf, blockManager); // Init ErasureCodingPolicyManager instance. ErasureCodingPolicyManager.getInstance().init(conf); @@ -1040,6 +1060,10 @@ public int getMaxListOpenFilesResponses() { return maxListOpenFilesResponses; } + boolean isSnapshotTrashRootEnabled() { + return isSnapshotTrashRootEnabled; + } + void lockRetryCache() { if (retryCache != null) { retryCache.lock(); @@ -1359,6 +1383,10 @@ void startActiveServices() throws IOException { dir.enableQuotaChecks(); dir.ezManager.startReencryptThreads(); + if (snapshotDeletionGc != null) { + snapshotDeletionGc.schedule(); + } + if (haEnabled) { // Renew all of the leases before becoming active. // This is because, while we were in standby mode, @@ -1543,6 +1571,12 @@ public void checkOperation(OperationCategory op) throws StandbyException { // null in some unit tests haContext.checkOperation(op); } + + boolean assertsEnabled = false; + assert assertsEnabled = true; // Intentional side effect!!! + if (assertsEnabled && op == OperationCategory.WRITE) { + getSnapshotManager().initThreadLocals(); + } } /** @@ -2260,7 +2294,7 @@ boolean truncate(String src, long newLength, String clientName, requireEffectiveLayoutVersionForFeature(Feature.TRUNCATE); FSDirTruncateOp.TruncateResult r = null; try { - NameNode.stateChangeLog.debug( + NameNode.stateChangeLog.info( "DIR* NameSystem.truncate: src={} newLength={}", src, newLength); if (newLength < 0) { throw new HadoopIllegalArgumentException( @@ -3287,10 +3321,10 @@ boolean delete(String src, boolean recursive, boolean logRetryCache) throw e; } getEditLog().logSync(); + logAuditEvent(ret, operationName, src); if (toRemovedBlocks != null) { removeBlocks(toRemovedBlocks); // Incremental deletion of blocks } - logAuditEvent(true, operationName, src); return ret; } @@ -5479,6 +5513,9 @@ private void registerMBean() { * Shutdown FSNamesystem. */ void shutdown() { + if (snapshotDeletionGc != null) { + snapshotDeletionGc.cancel(); + } if (snapshotManager != null) { snapshotManager.shutdown(); } @@ -7001,7 +7038,38 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() logAuditEvent(true, operationName, null, null, null); return status; } - + + /** + * Get the list of snapshots for a given snapshottable directory. + * + * @return The list of all the snapshots for a snapshottable directory + * @throws IOException + */ + public SnapshotStatus[] getSnapshotListing(String snapshotRoot) + throws IOException { + final String operationName = "ListSnapshot"; + SnapshotStatus[] status; + checkOperation(OperationCategory.READ); + boolean success = false; + final FSPermissionChecker pc = getPermissionChecker(); + FSPermissionChecker.setOperationType(operationName); + try { + readLock(); + try { + checkOperation(OperationCategory.READ); + status = FSDirSnapshotOp.getSnapshotListing(dir, pc, snapshotManager, + snapshotRoot); + success = true; + } finally { + readUnlock(operationName, getLockReportInfoSupplier(null)); + } + } catch (AccessControlException ace) { + logAuditEvent(success, operationName, snapshotRoot); + throw ace; + } + logAuditEvent(success, operationName, snapshotRoot); + return status; + } /** * Get the difference between two snapshots (or between a snapshot and the * current status) of a snapshottable directory. @@ -7166,6 +7234,30 @@ void deleteSnapshot(String snapshotRoot, String snapshotName, logAuditEvent(true, operationName, rootPath, null, null); } + public void gcDeletedSnapshot(String snapshotRoot, String snapshotName) + throws IOException { + final String operationName = "gcDeletedSnapshot"; + String rootPath = null; + final INode.BlocksMapUpdateInfo blocksToBeDeleted; + + checkOperation(OperationCategory.WRITE); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); + rootPath = Snapshot.getSnapshotPath(snapshotRoot, snapshotName); + checkNameNodeSafeMode("Cannot gcDeletedSnapshot for " + rootPath); + + final long now = Time.now(); + final INodesInPath iip = dir.resolvePath(null, snapshotRoot, DirOp.WRITE); + snapshotManager.assertMarkedAsDeleted(iip, snapshotName); + blocksToBeDeleted = FSDirSnapshotOp.deleteSnapshot( + dir, snapshotManager, iip, snapshotName, now, snapshotRoot, false); + } finally { + writeUnlock(operationName, getLockReportInfoSupplier(rootPath)); + } + removeBlocks(blocksToBeDeleted); + } + /** * Remove a list of INodeDirectorySnapshottable from the SnapshotManager * @param toRemove the list of INodeDirectorySnapshottable to be removed @@ -8434,6 +8526,32 @@ void checkAccess(String src, FsAction mode) throws IOException { logAuditEvent(true, operationName, src); } + /** + * Check if snapshot roots are created for all existing snapshottable + * directories. Create them if not. + */ + void checkAndProvisionSnapshotTrashRoots() throws IOException { + SnapshottableDirectoryStatus[] dirStatusList = getSnapshottableDirListing(); + if (dirStatusList == null) { + return; + } + for (SnapshottableDirectoryStatus dirStatus : dirStatusList) { + String currDir = dirStatus.getFullPath().toString(); + if (!currDir.endsWith(Path.SEPARATOR)) { + currDir += Path.SEPARATOR; + } + String trashPath = currDir + FileSystem.TRASH_PREFIX; + HdfsFileStatus fileStatus = getFileInfo(trashPath, false, false, false); + if (fileStatus == null) { + LOG.info("Trash doesn't exist for snapshottable directory {}. " + + "Creating trash at {}", currDir, trashPath); + PermissionStatus permissionStatus = new PermissionStatus(getRemoteUser() + .getShortUserName(), null, SHARED_TRASH_PERMISSION); + mkdirs(trashPath, permissionStatus, false); + } + } + } + private Supplier getLockReportInfoSupplier(String src) { return getLockReportInfoSupplier(src, null); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java index 8ed17bcccc8d3..c03cfd50756b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java @@ -26,7 +26,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Supplier; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.log.LogThrottlingHelper; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java index 93fefeea317d0..a83ec51529b50 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java @@ -24,7 +24,7 @@ import java.util.List; import java.util.Stack; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.ipc.CallerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java index 2acbda4005b57..a90dc27a54fa8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java @@ -34,7 +34,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * FSTreeTraverser traverse directory recursively and process files diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java index 78394ab1520e5..9b6f82f088450 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java @@ -45,11 +45,11 @@ import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import org.apache.hadoop.io.nativeio.NativeIO; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.ComparisonChain; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; /** * Journal manager for the common case of edits files being written diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java index 5dcb5069e870c..3b8c33165395b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java @@ -28,8 +28,12 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; +import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; +import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile; import org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics; +import org.apache.hadoop.hdfs.server.namenode.visitor.INodeCountVisitor; +import org.apache.hadoop.hdfs.server.namenode.visitor.INodeCountVisitor.Counts; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.util.GSet; import org.apache.hadoop.util.StringUtils; @@ -40,15 +44,21 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; import java.util.Timer; import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicInteger; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ENABLE_RETRY_CACHE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY; +import static org.apache.hadoop.hdfs.server.namenode.FsImageValidation.Cli.println; import static org.apache.hadoop.util.Time.now; /** @@ -134,6 +144,25 @@ static String toCommaSeparatedNumber(long n) { } return b.insert(0, n).toString(); } + + /** @return a filter for the given type. */ + static FilenameFilter newFilenameFilter(NameNodeFile type) { + final String prefix = type.getName() + "_"; + return new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + if (!name.startsWith(prefix)) { + return false; + } + for (int i = prefix.length(); i < name.length(); i++) { + if (!Character.isDigit(name.charAt(i))) { + return false; + } + } + return true; + } + }; + } } private final File fsImageFile; @@ -142,21 +171,44 @@ static String toCommaSeparatedNumber(long n) { this.fsImageFile = fsImageFile; } - int checkINodeReference(Configuration conf) throws Exception { + int run() throws Exception { + return run(new Configuration(), new AtomicInteger()); + } + + int run(AtomicInteger errorCount) throws Exception { + return run(new Configuration(), errorCount); + } + + int run(Configuration conf, AtomicInteger errorCount) throws Exception { + final int initCount = errorCount.get(); LOG.info(Util.memoryInfo()); initConf(conf); + // check INodeReference + final FSNamesystem namesystem = checkINodeReference(conf, errorCount); + + // check INodeMap + INodeMapValidation.run(namesystem.getFSDirectory(), errorCount); + LOG.info(Util.memoryInfo()); + + final int d = errorCount.get() - initCount; + if (d > 0) { + Cli.println("Found %d error(s) in %s", d, fsImageFile.getAbsolutePath()); + } + return d; + } + + private FSNamesystem loadImage(Configuration conf) throws IOException { final TimerTask checkProgress = new TimerTask() { @Override public void run() { final double percent = NameNode.getStartupProgress().createView() .getPercentComplete(Phase.LOADING_FSIMAGE); - LOG.info(String.format("%s Progress: %.1f%%", - Phase.LOADING_FSIMAGE, 100*percent)); + LOG.info(String.format("%s Progress: %.1f%% (%s)", + Phase.LOADING_FSIMAGE, 100*percent, Util.memoryInfo())); } }; - INodeReferenceValidation.start(); final Timer t = new Timer(); t.scheduleAtFixedRate(checkProgress, 0, 60_000); final long loadStart = now(); @@ -197,10 +249,42 @@ public void run() { t.cancel(); Cli.println("Loaded %s %s successfully in %s", FS_IMAGE, fsImageFile, StringUtils.formatTime(now() - loadStart)); + return namesystem; + } + + FSNamesystem checkINodeReference(Configuration conf, + AtomicInteger errorCount) throws Exception { + INodeReferenceValidation.start(); + final FSNamesystem namesystem = loadImage(conf); LOG.info(Util.memoryInfo()); - final int errorCount = INodeReferenceValidation.end(); + INodeReferenceValidation.end(errorCount); LOG.info(Util.memoryInfo()); - return errorCount; + return namesystem; + } + + static class INodeMapValidation { + static Iterable iterate(INodeMap map) { + return new Iterable() { + @Override + public Iterator iterator() { + return map.getMapIterator(); + } + }; + } + + static void run(FSDirectory fsdir, AtomicInteger errorCount) { + final int initErrorCount = errorCount.get(); + final Counts counts = INodeCountVisitor.countTree(fsdir.getRoot()); + for (INodeWithAdditionalFields i : iterate(fsdir.getINodeMap())) { + if (counts.getCount(i) == 0) { + Cli.printError(errorCount, "%s (%d) is inaccessible (%s)", + i, i.getId(), i.getFullPathName()); + } + } + println("%s ended successfully: %d error(s) found.", + INodeMapValidation.class.getSimpleName(), + errorCount.get() - initErrorCount); + } } static class Cli extends Configured implements Tool { @@ -217,9 +301,10 @@ public int run(String[] args) throws Exception { initLogLevels(); final FsImageValidation validation = FsImageValidation.newInstance(args); - final int errorCount = validation.checkINodeReference(getConf()); + final AtomicInteger errorCount = new AtomicInteger(); + validation.run(getConf(), errorCount); println("Error Count: %s", errorCount); - return errorCount == 0? 0: 1; + return errorCount.get() == 0? 0: 1; } static String parse(String... args) { @@ -240,19 +325,68 @@ static String parse(String... args) { return f; } - static void println(String format, Object... args) { + static synchronized void println(String format, Object... args) { final String s = String.format(format, args); System.out.println(s); LOG.info(s); } - static void printError(String message, Throwable t) { + static synchronized void warn(String format, Object... args) { + final String s = "WARN: " + String.format(format, args); + System.out.println(s); + LOG.warn(s); + } + + static synchronized void printError(String message, Throwable t) { System.out.println(message); if (t != null) { t.printStackTrace(System.out); } LOG.error(message, t); } + + static synchronized void printError(AtomicInteger errorCount, + String format, Object... args) { + final int count = errorCount.incrementAndGet(); + final String s = "FSIMAGE_ERROR " + count + ": " + + String.format(format, args); + System.out.println(s); + LOG.info(s); + } + } + + public static int validate(FSNamesystem namesystem) throws Exception { + final AtomicInteger errorCount = new AtomicInteger(); + final NNStorage nnStorage = namesystem.getFSImage().getStorage(); + for(Storage.StorageDirectory sd : nnStorage.getStorageDirs()) { + validate(sd.getCurrentDir(), errorCount); + } + return errorCount.get(); + } + + public static void validate(File path, AtomicInteger errorCount) + throws Exception { + if (path.isFile()) { + new FsImageValidation(path).run(errorCount); + } else if (path.isDirectory()) { + final File[] images = path.listFiles( + Util.newFilenameFilter(NameNodeFile.IMAGE)); + if (images == null || images.length == 0) { + Cli.warn("%s not found in %s", FSImage.class.getSimpleName(), + path.getAbsolutePath()); + return; + } + + Arrays.sort(images, Collections.reverseOrder()); + for (int i = 0; i < images.length; i++) { + final File image = images[i]; + Cli.println("%s %d) %s", FSImage.class.getSimpleName(), + i, image.getAbsolutePath()); + FsImageValidation.validate(image, errorCount); + } + } + + Cli.warn("%s is neither a file nor a directory", path.getAbsolutePath()); } public static void main(String[] args) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index fd416f72a3a20..3752117d4b367 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.Path; @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithName; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor; import org.apache.hadoop.hdfs.util.Diff; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.util.ChunkedArrayList; @@ -76,7 +77,7 @@ final boolean isRoot() { } /** Get the {@link PermissionStatus} */ - abstract PermissionStatus getPermissionStatus(int snapshotId); + public abstract PermissionStatus getPermissionStatus(int snapshotId); /** The same as getPermissionStatus(null). */ final PermissionStatus getPermissionStatus() { @@ -1123,6 +1124,14 @@ public void clear() { } } + /** Accept a visitor to visit this {@link INode}. */ + public void accept(NamespaceVisitor visitor, int snapshot) { + final Class clazz = visitor != null? visitor.getClass() + : NamespaceVisitor.class; + throw new UnsupportedOperationException(getClass().getSimpleName() + + " does not support " + clazz.getSimpleName()); + } + /** * INode feature such as {@link FileUnderConstructionFeature} * and {@link DirectoryWithQuotaFeature}. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java index 63c5b46b2fbf0..e83c962a4a845 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index de710a1c876bc..9845332c292aa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -34,14 +34,16 @@ import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; +import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager; import org.apache.hadoop.hdfs.util.ReadOnlyList; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.security.AccessControlException; import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; @@ -281,12 +283,11 @@ public void setSnapshotQuota(int snapshotQuota) { * @param name Name of the snapshot. * @param mtime The snapshot creation time set by Time.now(). */ - public Snapshot addSnapshot(int id, String name, - final LeaseManager leaseManager, final boolean captureOpenFiles, - int maxSnapshotLimit, long mtime) + public Snapshot addSnapshot(SnapshotManager snapshotManager, String name, + final LeaseManager leaseManager, long mtime) throws SnapshotException { - return getDirectorySnapshottableFeature().addSnapshot(this, id, name, - leaseManager, captureOpenFiles, maxSnapshotLimit, mtime); + return getDirectorySnapshottableFeature().addSnapshot(this, + snapshotManager, name, leaseManager, mtime); } /** @@ -294,11 +295,11 @@ public Snapshot addSnapshot(int id, String name, * @param snapshotName Name of the snapshot. * @param mtime The snapshot deletion time set by Time.now(). */ - public Snapshot removeSnapshot( - ReclaimContext reclaimContext, String snapshotName, long mtime) + public Snapshot removeSnapshot(ReclaimContext reclaimContext, + String snapshotName, long mtime, SnapshotManager snapshotManager) throws SnapshotException { return getDirectorySnapshottableFeature().removeSnapshot( - reclaimContext, this, snapshotName, mtime); + reclaimContext, this, snapshotName, mtime, snapshotManager); } /** @@ -995,6 +996,11 @@ public SnapshotAndINode(int snapshot, INode inode) { } } + @Override + public void accept(NamespaceVisitor visitor, int snapshot) { + visitor.visitDirectoryRecursively(this, snapshot); + } + public final int getChildrenNum(final int snapshotId) { return getChildrenList(snapshotId).size(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java index 240aa15ee5fb5..5e5c4b4b81fb7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java @@ -22,7 +22,7 @@ import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.util.EnumCounters; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * The attributes of an inode. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 3f0208c4dfb0c..2b2d4cfa99734 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -53,12 +53,13 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.DiffList; +import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor; import org.apache.hadoop.hdfs.util.LongBitFormat; import org.apache.hadoop.util.StringUtils; import static org.apache.hadoop.io.erasurecode.ErasureCodeConstants.REPLICATION_POLICY_ID; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** I-node for closed file. */ @InterfaceAudience.Private @@ -1111,6 +1112,11 @@ public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix, out.println(); } + @Override + public void accept(NamespaceVisitor visitor, int snapshot) { + visitor.visitFile(this, snapshot); + } + /** * Remove full blocks at the end file up to newLength * @return sum of sizes of the remained blocks diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java index bc273d28d7f99..f35949fdcdbed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java @@ -26,7 +26,7 @@ import org.apache.hadoop.util.GSet; import org.apache.hadoop.util.LightWeightGSet; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Storing all the {@link INode}s and maintaining the mapping between INode ID diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java index 69a92706ab50b..ccb4b87a0e8de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java @@ -30,7 +30,8 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor; import org.apache.hadoop.security.AccessControlException; /** @@ -368,7 +369,12 @@ public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix, b.append("->"); getReferredINode().dumpTreeRecursively(out, b, snapshot); } - + + @Override + public void accept(NamespaceVisitor visitor, int snapshot) { + visitor.visitReferenceRecursively(this, snapshot); + } + public int getDstSnapshotId() { return Snapshot.CURRENT_STATE_ID; } @@ -399,7 +405,7 @@ public WithCount(INodeReference parent, INode referred) { INodeReferenceValidation.add(this, WithCount.class); } - private String getCountDetails() { + public String getCountDetails() { final StringBuilder b = new StringBuilder("["); if (!withNameList.isEmpty()) { final Iterator i = withNameList.iterator(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java index d3faf43074161..9241644dcb9c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.server.namenode.FsImageValidation.Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,18 +46,20 @@ public class INodeReferenceValidation { public static void start() { INSTANCE.compareAndSet(null, new INodeReferenceValidation()); - println("Validation started"); + println("%s started", INodeReferenceValidation.class.getSimpleName()); } - public static int end() { + public static void end(AtomicInteger errorCount) { final INodeReferenceValidation instance = INSTANCE.getAndSet(null); if (instance == null) { - return 0; + return; } - final int errorCount = instance.assertReferences(); - println("Validation ended successfully: %d error(s) found.", errorCount); - return errorCount; + final int initCount = errorCount.get(); + instance.assertReferences(errorCount); + println("%s ended successfully: %d error(s) found.", + INodeReferenceValidation.class.getSimpleName(), + errorCount.get() - initCount); } static void add(REF ref, Class clazz) { @@ -153,7 +155,7 @@ ReferenceSet getReferences( throw new IllegalArgumentException("References not found for " + clazz); } - private int assertReferences() { + private void assertReferences(AtomicInteger errorCount) { final int p = Runtime.getRuntime().availableProcessors(); LOG.info("Available Processors: {}", p); final ExecutorService service = Executors.newFixedThreadPool(p); @@ -168,7 +170,6 @@ public void run() { final Timer t = new Timer(); t.scheduleAtFixedRate(checkProgress, 0, 1_000); - final AtomicInteger errorCount = new AtomicInteger(); try { dstReferences.submit(errorCount, service); withCounts.submit(errorCount, service); @@ -183,7 +184,6 @@ public void run() { service.shutdown(); t.cancel(); } - return errorCount.get(); } static List> createTasks( @@ -215,7 +215,7 @@ public Integer call() throws Exception { try { ref.assertReferences(); } catch (Throwable t) { - println("%d: %s", errorCount.incrementAndGet(), t); + printError(errorCount, "%s", t); } } return references.size(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java index c76bea090f165..45f4d86edd4e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor; /** * An {@link INode} representing a symbolic link. @@ -104,7 +105,13 @@ public ContentSummaryComputationContext computeContentSummary(int snapshotId, public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix, final int snapshot) { super.dumpTreeRecursively(out, prefix, snapshot); - out.println(); + out.print(" ~> "); + out.println(getSymlinkString()); + } + + @Override + public void accept(NamespaceVisitor visitor, int snapshot) { + visitor.visitSymlink(this, snapshot); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java index b7d2f2c1e5abc..a2c7804a83521 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hdfs.util.LongBitFormat; import org.apache.hadoop.util.LightWeightGSet.LinkedElement; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * {@link INode} with additional fields including id, name, permission, @@ -178,7 +178,7 @@ final void clonePermissionStatus(INodeWithAdditionalFields that) { } @Override - final PermissionStatus getPermissionStatus(int snapshotId) { + public final PermissionStatus getPermissionStatus(int snapshotId) { return new PermissionStatus(getUserName(snapshotId), getGroupName(snapshotId), getFsPermission(snapshotId)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java index cc5eaa0ebe838..8a150f0630f6b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.CURRENT_STATE_ID; import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.ID_INTEGER_COMPARATOR; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java index a9c2a09ed48af..54c87383c7d36 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java @@ -63,8 +63,8 @@ import org.apache.hadoop.util.ServletUtil; import org.apache.hadoop.util.StringUtils; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * This class is used in Namesystem's jetty to retrieve/upload a file @@ -112,12 +112,25 @@ static void setRecentImageCheckTimePrecision(double ratio) { recentImageCheckTimePrecision = ratio; } + private FSImage getAndValidateFSImage(ServletContext context, + final HttpServletResponse response) + throws IOException { + final FSImage nnImage = NameNodeHttpServer.getFsImageFromContext(context); + if (nnImage == null) { + String errorMsg = "NameNode initialization not yet complete. " + + "FSImage has not been set in the NameNode."; + response.sendError(HttpServletResponse.SC_FORBIDDEN, errorMsg); + throw new IOException(errorMsg); + } + return nnImage; + } + @Override public void doGet(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException { try { final ServletContext context = getServletContext(); - final FSImage nnImage = NameNodeHttpServer.getFsImageFromContext(context); + final FSImage nnImage = getAndValidateFSImage(context, response); final GetImageParams parsedParams = new GetImageParams(request, response); final Configuration conf = (Configuration) context .getAttribute(JspHelper.CURRENT_CONF); @@ -524,7 +537,7 @@ protected void doPut(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException { try { ServletContext context = getServletContext(); - final FSImage nnImage = NameNodeHttpServer.getFsImageFromContext(context); + final FSImage nnImage = getAndValidateFSImage(context, response); final Configuration conf = (Configuration) getServletContext() .getAttribute(JspHelper.CURRENT_CONF); final PutImageParams parsedParams = new PutImageParams(request, response, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java index 8a54c8a775945..ba40da4d843b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.inotify.Event; import org.apache.hadoop.hdfs.inotify.EventBatch; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java index eb3a140ba4e0a..e2b7b1c68bad4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java @@ -40,10 +40,10 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; /** * Manages a collection of Journals. None of the methods are synchronized, it is diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java index c3624ce303385..baa56bb540320 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java @@ -37,7 +37,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; @@ -52,8 +52,8 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.util.Daemon; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java index 98ae44ede937d..db08ac200b670 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java @@ -57,9 +57,9 @@ import org.apache.hadoop.util.Time; import org.eclipse.jetty.util.ajax.JSON; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * NNStorage is responsible for management of the StorageDirectories used by diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java index aaaaa7210c13b..22be54e5576d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java @@ -37,10 +37,10 @@ import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile; import org.apache.hadoop.hdfs.util.MD5FileUtils; -import com.google.common.base.Preconditions; -import com.google.common.collect.ComparisonChain; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; /** * The NNStorageRetentionManager is responsible for inspecting the storage diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java index 9cca97ab74b32..8086b60637dae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.common.StorageInfo; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; public abstract class NNUpgradeUtil { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 9bffaaabebeaf..2f04ecd71d377 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -17,11 +17,11 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import java.util.Set; import org.apache.commons.logging.Log; @@ -727,11 +727,6 @@ protected void initialize(Configuration conf) throws IOException { intervals); } } - // Currently NN uses FileSystem.get to initialize DFS in startTrashEmptier. - // If fs.hdfs.impl was overridden by core-site.xml, we may get other - // filesystem. To make sure we get DFS, we are setting fs.hdfs.impl to DFS. - // HDFS-15450 - conf.set(FS_HDFS_IMPL_KEY, DistributedFileSystem.class.getName()); UserGroupInformation.setConfiguration(conf); loginAsNameNodeUser(conf); @@ -933,7 +928,9 @@ private void startTrashEmptier(final Configuration conf) throws IOException { new PrivilegedExceptionAction() { @Override public FileSystem run() throws IOException { - return FileSystem.get(conf); + FileSystem dfs = new DistributedFileSystem(); + dfs.initialize(FileSystem.getDefaultUri(conf), conf); + return dfs; } }); this.emptier = new Thread(new Trash(fs, conf).getEmptier(), "Trash Emptier"); @@ -2021,6 +2018,9 @@ public HAState getState() { public void startActiveServices() throws IOException { try { namesystem.startActiveServices(); + if (namesystem.isSnapshotTrashRootEnabled()) { + namesystem.checkAndProvisionSnapshotTrashRoots(); + } startTrashEmptier(getConf()); } catch (Throwable t) { doImmediateShutdown(t); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index 1a3b578510b8f..33913227af2ce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -27,7 +27,7 @@ import javax.servlet.ServletContext; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ha.HAServiceProtocol; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java index de8734f15efd9..4cac0feffdfa1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java @@ -33,7 +33,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.server.common.Util; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 230e4020117f0..eeee350a4acfa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -45,7 +45,7 @@ import java.util.Map; import java.util.Set; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; @@ -132,6 +132,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException; import org.apache.hadoop.hdfs.protocol.UnresolvedPathException; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ClientNamenodeProtocol; @@ -226,7 +227,7 @@ import org.apache.hadoop.util.VersionUtil; import org.slf4j.Logger; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.protobuf.BlockingService; import javax.annotation.Nonnull; @@ -1099,10 +1100,8 @@ public void rename2(String src, String dst, Options.Rename... options) public boolean truncate(String src, long newLength, String clientName) throws IOException { checkNNStartup(); - if(stateChangeLog.isDebugEnabled()) { - stateChangeLog.debug("*DIR* NameNode.truncate: " + src + " to " + - newLength); - } + stateChangeLog + .debug("*DIR* NameNode.truncate: " + src + " to " + newLength); String clientMachine = getClientMachine(); try { return namesystem.truncate( @@ -2004,6 +2003,16 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() return status; } + @Override // Client Protocol + public SnapshotStatus[] getSnapshotListing(String snapshotRoot) + throws IOException { + checkNNStartup(); + SnapshotStatus[] status = namesystem + .getSnapshotListing(snapshotRoot); + metrics.incrListSnapshotsOps(); + return status; + } + @Override // ClientProtocol public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot, String earlierSnapshotName, String laterSnapshotName) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeUtils.java index 837b1a9fa9e97..1eae13e467980 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeUtils.java @@ -20,7 +20,7 @@ import javax.annotation.Nullable; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSUtilClient; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index 469ef893d7658..370c1b2b48137 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -89,7 +89,7 @@ import org.apache.hadoop.util.Time; import org.apache.htrace.core.Tracer; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * This class provides rudimentary checking of DFS volumes for errors and diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaByStorageTypeEntry.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaByStorageTypeEntry.java index 934c3169fc014..fd98ce9ef6146 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaByStorageTypeEntry.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaByStorageTypeEntry.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.base.Objects; +import org.apache.hadoop.thirdparty.com.google.common.base.Objects; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.util.StringUtils; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java index bcb21929c8b9f..002972caf63e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.util.ConstEnumCounters; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java index cbdd292a3cdb9..15f799ab215ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java @@ -26,8 +26,8 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.io.IOUtils; -import com.google.common.base.Preconditions; -import com.google.common.primitives.Longs; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Longs; import org.apache.hadoop.log.LogThrottlingHelper; import org.apache.hadoop.log.LogThrottlingHelper.LogAction; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java index fd9cbd752750e..ea38da6021ce8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java index 15cfa9278f6b9..f2d09b0627f3d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java index 823385a980667..35a7cd2f643cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.util.Canceler; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Context for an ongoing SaveNamespace operation. This class diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index 1423b308012c0..29cfe4280287b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -29,7 +29,7 @@ import java.security.PrivilegedExceptionAction; import java.util.*; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.HelpFormatter; @@ -77,8 +77,8 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.util.VersionInfo; import javax.management.ObjectName; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java index 82cdcfe69d9be..7ba6d839bea1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java @@ -57,8 +57,8 @@ import org.apache.hadoop.util.Time; import org.apache.http.client.utils.URIBuilder; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.eclipse.jetty.io.EofException; import static org.apache.hadoop.hdfs.server.common.Util.IO_FILE_BUFFER_SIZE; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFeature.java index a7f08780a63ec..73df932f2fbe5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFeature.java @@ -24,7 +24,7 @@ import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.hdfs.XAttrHelper; -import com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; /** * Feature for extended attributes. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java index 5b5992c7a5894..af1025ab457ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java @@ -25,8 +25,8 @@ import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.hdfs.XAttrHelper; -import com.google.common.base.Preconditions; -import com.google.common.primitives.Ints; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; import org.apache.hadoop.hdfs.util.LongBitFormat; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java index b6f4f64ca517c..92e5ef1a0b86d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java @@ -25,8 +25,8 @@ import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.security.AccessControlException; -import com.google.common.collect.Lists; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java index 0babaaf0c1bc5..3f273cb5e75e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java @@ -70,8 +70,8 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Tool which allows the standby node's storage directories to be bootstrapped diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java index d32bcece33faa..b82fb5b0e41d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java @@ -34,8 +34,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import com.google.common.collect.Iterators; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.collect.Iterators; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -58,8 +58,8 @@ import static org.apache.hadoop.util.Time.monotonicNow; import static org.apache.hadoop.util.ExitUtil.terminate; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.util.Time; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RemoteNameNodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RemoteNameNodeInfo.java index 9a51190b17607..1ff251d808c8c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RemoteNameNodeInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RemoteNameNodeInfo.java @@ -24,13 +24,13 @@ import java.util.Collections; import java.util.List; -import com.google.common.base.Objects; +import org.apache.hadoop.thirdparty.com.google.common.base.Objects; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.server.namenode.NameNode; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Information about a single remote NameNode diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java index daa836ac69a5f..1a86f8e82f7dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java @@ -19,7 +19,7 @@ import static org.apache.hadoop.util.Time.monotonicNow; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import java.io.IOException; import java.net.URI; import java.net.URL; @@ -48,9 +48,9 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java index c15cdbdd48e4e..de99ddfaa92d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java @@ -77,6 +77,8 @@ public class NameNodeMetrics { MutableCounterLong renameSnapshotOps; @Metric("Number of listSnapshottableDirectory operations") MutableCounterLong listSnapshottableDirOps; + @Metric("Number of listSnapshots operations") + MutableCounterLong listSnapshotOps; @Metric("Number of snapshotDiffReport operations") MutableCounterLong snapshotDiffReportOps; @Metric("Number of blockReceivedAndDeleted calls") @@ -106,6 +108,7 @@ public long totalFileOps(){ disallowSnapshotOps.value() + renameSnapshotOps.value() + listSnapshottableDirOps.value() + + listSnapshotOps.value() + createSymlinkOps.value() + snapshotDiffReportOps.value(); } @@ -319,6 +322,10 @@ public void incrRenameSnapshotOps() { public void incrListSnapshottableDirOps() { listSnapshottableDirOps.incr(); } + + public void incrListSnapshotsOps() { + listSnapshotOps.incr(); + } public void incrSnapshotDiffReportOps() { snapshotDiffReportOps.incr(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java index 2e8620b649593..df052f171afa8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeAttributes; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * The difference of an inode between in two snapshots. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java index 163b181ed1892..16e3b7536918e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java @@ -76,7 +76,11 @@ public final void deleteSnapshotDiff(INode.ReclaimContext reclaimContext, if (diffs == null) { return; } - int snapshotIndex = diffs.binarySearch(snapshot); + final int snapshotIndex = diffs.binarySearch(snapshot); + // DeletionOrdered: only can remove the element at index 0 and no prior + // check snapshotIndex <= 0 since the diff may not exist + assert !SnapshotManager.isDeletionOrdered() + || (snapshotIndex <= 0 && prior == Snapshot.NO_SNAPSHOT_ID); D removed; if (snapshotIndex == 0) { @@ -133,6 +137,17 @@ final void addFirst(D diff) { diff.setPosterior(first); } + /** @return the first diff. */ + final D getFirst() { + return diffs == null || diffs.isEmpty()? null: diffs.get(0); + } + + /** @return the first snapshot INode. */ + final A getFirstSnapshotINode() { + final D first = getFirst(); + return first == null? null: first.getSnapshotINode(); + } + /** @return the last diff. */ public final D getLast() { if (diffs == null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java index 95c23dfc02e27..7fa3f05efd0d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java @@ -57,6 +57,8 @@ public int size() { @Override public T remove(int i) { + // DeletionOrdered: only can remove the element at index 0 + assert !SnapshotManager.isDeletionOrdered() || i == 0; return list.remove(i); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java index 705b8d6937b52..dedc1e49d341f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode.snapshot; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.snapshot. DirectoryWithSnapshotFeature.DirectoryDiff; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java index b38d8bfe8ce06..affdb1ba3ae66 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java @@ -47,9 +47,9 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.util.Time; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; /** * A directory with this feature is a snapshottable directory, where snapshots @@ -172,27 +172,25 @@ void addSnapshot(Snapshot snapshot) { /** * Add a snapshot. * @param snapshotRoot Root of the snapshot. + * @param snapshotManager SnapshotManager Instance. * @param name Name of the snapshot. - * @param mtime The snapshot creation time set by Time.now(). + * @param leaseManager * @throws SnapshotException Throw SnapshotException when there is a snapshot * with the same name already exists or snapshot quota exceeds */ - public Snapshot addSnapshot(INodeDirectory snapshotRoot, int id, String name, - final LeaseManager leaseManager, final boolean captureOpenFiles, - int maxSnapshotLimit, long now) + public Snapshot addSnapshot(INodeDirectory snapshotRoot, + SnapshotManager snapshotManager, String name, + final LeaseManager leaseManager, long now) throws SnapshotException { + int id = snapshotManager.getSnapshotCounter(); //check snapshot quota final int n = getNumSnapshots(); if (n + 1 > snapshotQuota) { throw new SnapshotException("Failed to add snapshot: there are already " + n + " snapshot(s) and the snapshot quota is " + snapshotQuota); - } else if (n + 1 > maxSnapshotLimit) { - throw new SnapshotException( - "Failed to add snapshot: there are already " + n - + " snapshot(s) and the max snapshot limit is " - + maxSnapshotLimit); } + snapshotManager.checkPerDirectorySnapshotLimit(n); final Snapshot s = new Snapshot(id, name, snapshotRoot); final byte[] nameBytes = s.getRoot().getLocalNameBytes(); final int i = searchSnapshot(nameBytes); @@ -209,7 +207,7 @@ public Snapshot addSnapshot(INodeDirectory snapshotRoot, int id, String name, snapshotRoot.updateModificationTime(now, Snapshot.CURRENT_STATE_ID); s.getRoot().setModificationTime(now, Snapshot.CURRENT_STATE_ID); - if (captureOpenFiles) { + if (snapshotManager.captureOpenFiles()) { try { Set openFilesIIP = leaseManager.getINodeWithLeases(snapshotRoot); @@ -239,15 +237,35 @@ public Snapshot addSnapshot(INodeDirectory snapshotRoot, int id, String name, */ public Snapshot removeSnapshot( INode.ReclaimContext reclaimContext, INodeDirectory snapshotRoot, - String snapshotName, long now) throws SnapshotException { + String snapshotName, long now, SnapshotManager snapshotManager) + throws SnapshotException { final int i = searchSnapshot(DFSUtil.string2Bytes(snapshotName)); if (i < 0) { + // considering a sequence like this with snapshots S1 and s2 + // 1. Ordered snapshot deletion feature is turned on + // 2. Delete S2 creating edit log entry for S2 deletion + // 3. Delete S1 + // 4. S2 gets deleted by snapshot gc thread creating edit log record for + // S2 deletion again + // 5. Disable Ordered snapshot deletion feature + // 6. Restarting Namenode + // In this case, when edit log replay happens actual deletion of S2 + // will happen when first edit log for S2 deletion gets replayed and + // the second edit log record replay for S2 deletion will fail as snapshot + // won't exist thereby failing the Namenode start + // The idea here is to check during edit log replay, if a certain snapshot + // is not found and the ordered snapshot deletion is off, ignore the error + if (!snapshotManager.isSnapshotDeletionOrdered() && + !snapshotManager.isImageLoaded()) { + return null; + } throw new SnapshotException("Cannot delete snapshot " + snapshotName + " from path " + snapshotRoot.getFullPathName() + ": the snapshot does not exist."); } else { final Snapshot snapshot = snapshotsByNames.get(i); int prior = Snapshot.findLatestSnapshot(snapshotRoot, snapshot.getId()); + snapshotManager.assertPrior(snapshotRoot, snapshotName, prior); snapshotRoot.cleanSubtree(reclaimContext, snapshot.getId(), prior); // remove from snapshotsByNames after successfully cleaning the subtree snapshotsByNames.remove(i); @@ -358,7 +376,7 @@ SnapshotDiffListingInfo computeDiff(final INodeDirectory snapshotRootDir, * @throws SnapshotException If snapshotName is not null or empty, but there * is no snapshot matching the name. */ - private Snapshot getSnapshotByName(INodeDirectory snapshotRoot, + public Snapshot getSnapshotByName(INodeDirectory snapshotRoot, String snapshotName) throws SnapshotException { Snapshot s = null; if (snapshotName != null && !snapshotName.isEmpty()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java index f76738ffa2924..2c0c8146f4044 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode.snapshot; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.namenode.*; @@ -166,7 +166,7 @@ void setSnapshotRoot(INodeDirectoryAttributes root) { this.isSnapshotRoot = true; } - boolean isSnapshotRoot() { + public boolean isSnapshotRoot() { return isSnapshotRoot; } @@ -175,6 +175,8 @@ void combinePosteriorAndCollectBlocks( final INode.ReclaimContext reclaimContext, final INodeDirectory currentDir, final DirectoryDiff posterior) { + // DeletionOrdered: must not combine posterior + assert !SnapshotManager.isDeletionOrdered(); diff.combinePosterior(posterior.diff, new Diff.Processor() { /** Collect blocks for deleted files. */ @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java index 5f5cd41166a7e..a6e2d5debb13b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java @@ -35,7 +35,7 @@ import java.util.List; import java.util.Map; -import com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.fs.StorageType; @@ -82,7 +82,7 @@ import org.apache.hadoop.hdfs.server.namenode.XAttrFeature; import org.apache.hadoop.hdfs.util.EnumCounters; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.thirdparty.protobuf.ByteString; @InterfaceAudience.Private @@ -438,7 +438,7 @@ public void serializeSnapshotSection(OutputStream out) throws IOException { .setSnapshotCounter(sm.getSnapshotCounter()) .setNumSnapshots(sm.getNumSnapshots()); - INodeDirectory[] snapshottables = sm.getSnapshottableDirs(); + final List snapshottables = sm.getSnapshottableDirs(); for (INodeDirectory sdir : snapshottables) { b.addSnapshottableDir(sdir.getId()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java index f13ef611346a4..9a3ee2e6c4e61 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java @@ -40,6 +40,8 @@ import org.apache.hadoop.security.AccessControlException; +import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_SNAPSHOT_DELETED; + /** Snapshot of a sub-tree in the namesystem. */ @InterfaceAudience.Private public class Snapshot implements Comparable { @@ -156,6 +158,11 @@ static public class Root extends INodeDirectory { }).collect(Collectors.toList()).toArray(new Feature[0])); } + boolean isMarkedAsDeleted() { + final XAttrFeature f = getXAttrFeature(); + return f != null && f.getXAttr(XATTR_SNAPSHOT_DELETED) != null; + } + @Override public ReadOnlyList getChildrenList(int snapshotId) { return getParent().getChildrenList(snapshotId); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java new file mode 100644 index 0000000000000..d57da2204aa38 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.snapshot; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS; +import static org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS_DEFAULT; + +public class SnapshotDeletionGc { + public static final Logger LOG = LoggerFactory.getLogger( + SnapshotDeletionGc.class); + + private final FSNamesystem namesystem; + private final long deletionOrderedGcPeriodMs; + private final AtomicReference timer = new AtomicReference<>(); + + public SnapshotDeletionGc(FSNamesystem namesystem, Configuration conf) { + this.namesystem = namesystem; + + this.deletionOrderedGcPeriodMs = conf.getLong( + DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS, + DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS_DEFAULT); + LOG.info("{} = {}", DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS, + deletionOrderedGcPeriodMs); + } + + public void schedule() { + if (timer.get() != null) { + return; + } + final Timer t = new Timer(getClass().getSimpleName(), true); + if (timer.compareAndSet(null, t)) { + LOG.info("Schedule at fixed rate {}", + StringUtils.formatTime(deletionOrderedGcPeriodMs)); + t.scheduleAtFixedRate(new GcTask(), + deletionOrderedGcPeriodMs, deletionOrderedGcPeriodMs); + } + } + + public void cancel() { + final Timer t = timer.getAndSet(null); + if (t != null) { + LOG.info("cancel"); + t.cancel(); + } + } + + private void gcDeletedSnapshot(String name) { + final Snapshot.Root deleted; + namesystem.readLock(); + try { + deleted = namesystem.getSnapshotManager().chooseDeletedSnapshot(); + } catch (Throwable e) { + LOG.error("Failed to chooseDeletedSnapshot", e); + throw e; + } finally { + namesystem.readUnlock(); + } + if (deleted == null) { + LOG.trace("{}: no snapshots are marked as deleted.", name); + return; + } + + final String snapshotRoot = deleted.getRootFullPathName(); + final String snapshotName = deleted.getLocalName(); + LOG.info("{}: delete snapshot {} from {}", + name, snapshotName, snapshotRoot); + + try { + namesystem.gcDeletedSnapshot(snapshotRoot, snapshotName); + } catch (Throwable e) { + LOG.error("Failed to gcDeletedSnapshot " + deleted.getFullPathName(), e); + } + } + + private class GcTask extends TimerTask { + private final AtomicInteger count = new AtomicInteger(); + + @Override + public void run() { + final int id = count.incrementAndGet(); + gcDeletedSnapshot(getClass().getSimpleName() + " #" + id); + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java index ab6f4158c8417..58dd2cf0a3f04 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java @@ -33,8 +33,8 @@ import org.apache.hadoop.hdfs.server.namenode.INodeReference; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff; -import com.google.common.base.Preconditions; -import com.google.common.primitives.SignedBytes; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.primitives.SignedBytes; import org.apache.hadoop.util.ChunkedArrayList; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java index a7960700e6811..8861a1f0ecbdc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeReference; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.util.ChunkedArrayList; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java index d60a03822e57d..b43c45854bbfd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff; import org.apache.hadoop.hdfs.util.ReadOnlyList; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * A helper class defining static methods for reading/writing snapshot related diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index 30b98b8e86421..bb89c6e6285b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -31,23 +31,30 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import javax.management.ObjectName; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.XAttr; +import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtilClient; +import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.protocol.SnapshotInfo; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; -import org.apache.hadoop.hdfs.server.namenode.FSDirectory; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; +import org.apache.hadoop.hdfs.server.namenode.*; import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp; import org.apache.hadoop.hdfs.server.namenode.FSImageFormat; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -55,9 +62,10 @@ import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodesInPath; import org.apache.hadoop.hdfs.server.namenode.LeaseManager; +import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.metrics2.util.MBeans; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,6 +85,28 @@ public class SnapshotManager implements SnapshotStatsMXBean { public static final Logger LOG = LoggerFactory.getLogger(SnapshotManager.class); + // The following are private configurations + static final String DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED + = "dfs.namenode.snapshot.deletion.ordered"; + static final boolean DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_DEFAULT + = false; + static final String DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS + = "dfs.namenode.snapshot.deletion.ordered.gc.period.ms"; + static final long DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS_DEFAULT + = 5 * 60_000L; //5 minutes + + private static final ThreadLocal DELETION_ORDERED + = new ThreadLocal<>(); + + static boolean isDeletionOrdered() { + final Boolean b = DELETION_ORDERED.get(); + return b != null? b: false; + } + + public void initThreadLocals() { + DELETION_ORDERED.set(isSnapshotDeletionOrdered()); + } + private final FSDirectory fsdir; private boolean captureOpenFiles; /** @@ -99,14 +129,17 @@ public class SnapshotManager implements SnapshotStatsMXBean { private static final int SNAPSHOT_ID_BIT_WIDTH = 28; private boolean allowNestedSnapshots = false; + private final boolean snapshotDeletionOrdered; private int snapshotCounter = 0; private final int maxSnapshotLimit; + private final int maxSnapshotFSLimit; /** All snapshottable directories in the namesystem. */ private final Map snapshottables = - new HashMap(); + new ConcurrentHashMap<>(); - public SnapshotManager(final Configuration conf, final FSDirectory fsdir) { + public SnapshotManager(final Configuration conf, final FSDirectory fsdir) + throws SnapshotException { this.fsdir = fsdir; this.captureOpenFiles = conf.getBoolean( DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES, @@ -119,25 +152,49 @@ public SnapshotManager(final Configuration conf, final FSDirectory fsdir) { DFSConfigKeys. DFS_NAMENODE_SNAPSHOT_DIFF_ALLOW_SNAP_ROOT_DESCENDANT_DEFAULT); this.maxSnapshotLimit = conf.getInt( - DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_MAX_LIMIT, - DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_MAX_LIMIT_DEFAULT); + DFSConfigKeys. + DFS_NAMENODE_SNAPSHOT_MAX_LIMIT, + DFSConfigKeys. + DFS_NAMENODE_SNAPSHOT_MAX_LIMIT_DEFAULT); + this.maxSnapshotFSLimit = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_FILESYSTEM_LIMIT, + DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_FILESYSTEM_LIMIT_DEFAULT); LOG.info("Loaded config captureOpenFiles: " + captureOpenFiles + ", skipCaptureAccessTimeOnlyChange: " + skipCaptureAccessTimeOnlyChange + ", snapshotDiffAllowSnapRootDescendant: " + snapshotDiffAllowSnapRootDescendant + + ", maxSnapshotFSLimit: " + + maxSnapshotFSLimit + ", maxSnapshotLimit: " + maxSnapshotLimit); + this.snapshotDeletionOrdered = conf.getBoolean( + DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, + DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_DEFAULT); + LOG.info("{} = {}", DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, + snapshotDeletionOrdered); + final int maxLevels = conf.getInt( DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_SKIPLIST_MAX_LEVELS, DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_SKIPLIST_MAX_SKIP_LEVELS_DEFAULT); final int skipInterval = conf.getInt( DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_SKIPLIST_SKIP_INTERVAL, DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_SKIPLIST_SKIP_INTERVAL_DEFAULT); + if (maxSnapshotLimit > maxSnapshotFSLimit) { + final String errMsg = DFSConfigKeys. + DFS_NAMENODE_SNAPSHOT_MAX_LIMIT + + " cannot be greater than " + + DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_FILESYSTEM_LIMIT; + throw new SnapshotException(errMsg); + } DirectoryDiffListFactory.init(skipInterval, maxLevels, LOG); } + public boolean isSnapshotDeletionOrdered() { + return snapshotDeletionOrdered; + } + @VisibleForTesting void setCaptureOpenFiles(boolean captureOpenFiles) { this.captureOpenFiles = captureOpenFiles; @@ -268,6 +325,60 @@ public INodeDirectory getSnapshottableRoot(final INodesInPath iip) return dir; } + public void assertMarkedAsDeleted(INodesInPath iip, String snapshotName) + throws IOException { + final INodeDirectory dir = getSnapshottableRoot(iip); + final Snapshot.Root snapshotRoot = dir.getDirectorySnapshottableFeature() + .getSnapshotByName(dir, snapshotName) + .getRoot(); + + if (!snapshotRoot.isMarkedAsDeleted()) { + throw new SnapshotException("Failed to gcDeletedSnapshot " + + snapshotName + " from " + dir.getFullPathName() + + ": snapshot is not marked as deleted"); + } + } + + void assertPrior(INodeDirectory dir, String snapshotName, int prior) + throws SnapshotException { + if (!isSnapshotDeletionOrdered()) { + return; + } + // prior must not exist + if (prior != Snapshot.NO_SNAPSHOT_ID) { + throw new SnapshotException("Failed to removeSnapshot " + + snapshotName + " from " + dir.getFullPathName() + + ": Unexpected prior (=" + prior + ") when " + + DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED + + " is " + isSnapshotDeletionOrdered()); + } + } + + void assertFirstSnapshot(INodeDirectory dir, + DirectorySnapshottableFeature snapshottable, Snapshot snapshot) + throws SnapshotException { + final INodeDirectoryAttributes first + = snapshottable.getDiffs().getFirstSnapshotINode(); + if (snapshot.getRoot() != first) { + throw new SnapshotException("Failed to delete snapshot " + snapshot + + " from " + dir.getFullPathName() + " since " + snapshot + + " is not the first snapshot (=" + first + ") and " + + DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED + + " is " + isSnapshotDeletionOrdered()); + } + } + + /** + * Return CaptureOpenFiles config value. + */ + boolean captureOpenFiles() { + return captureOpenFiles; + } + + @VisibleForTesting + int getMaxSnapshotLimit() { + return maxSnapshotLimit; + } /** * Get the snapshot root directory for the given directory. The given * directory must either be a snapshot root or a descendant of any @@ -275,21 +386,35 @@ public INodeDirectory getSnapshottableRoot(final INodesInPath iip) * @param iip INodesInPath for the directory to get snapshot root. * @return the snapshot root INodeDirectory */ + public INodeDirectory checkAndGetSnapshottableAncestorDir( + final INodesInPath iip) throws IOException { + final INodeDirectory dir = getSnapshottableAncestorDir(iip); + if (dir == null) { + throw new SnapshotException("Directory is neither snapshottable nor" + + " under a snap root!"); + } + return dir; + } + public INodeDirectory getSnapshottableAncestorDir(final INodesInPath iip) throws IOException { final String path = iip.getPath(); - final INodeDirectory dir = INodeDirectory.valueOf(iip.getLastINode(), path); + final INode inode = iip.getLastINode(); + final INodeDirectory dir; + if (inode instanceof INodeDirectory) { + dir = INodeDirectory.valueOf(inode, path); + } else { + dir = INodeDirectory.valueOf(iip.getINode(-2), iip.getParentPath()); + } if (dir.isSnapshottable()) { return dir; - } else { - for (INodeDirectory snapRoot : this.snapshottables.values()) { - if (dir.isAncestorDirectory(snapRoot)) { - return snapRoot; - } + } + for (INodeDirectory snapRoot : this.snapshottables.values()) { + if (dir.isAncestorDirectory(snapRoot)) { + return snapRoot; } - throw new SnapshotException("Directory is neither snapshottable nor" + - " under a snap root!"); } + return null; } public boolean isDescendantOfSnapshotRoot(INodeDirectory dir) { @@ -333,16 +458,44 @@ public String createSnapshot(final LeaseManager leaseManager, "Failed to create the snapshot. The FileSystem has run out of " + "snapshot IDs and ID rollover is not supported."); } - - srcRoot.addSnapshot(snapshotCounter, snapshotName, leaseManager, - this.captureOpenFiles, maxSnapshotLimit, mtime); + int n = numSnapshots.get(); + checkFileSystemSnapshotLimit(n); + srcRoot.addSnapshot(this, snapshotName, leaseManager, mtime); //create success, update id snapshotCounter++; numSnapshots.getAndIncrement(); return Snapshot.getSnapshotPath(snapshotRoot, snapshotName); } - + + void checkFileSystemSnapshotLimit(int n) throws SnapshotException { + checkSnapshotLimit(maxSnapshotFSLimit, n, "file system"); + } + + void checkPerDirectorySnapshotLimit(int n) throws SnapshotException { + checkSnapshotLimit(maxSnapshotLimit, n, "per directory"); + } + + void checkSnapshotLimit(int limit, int snapshotCount, String type) + throws SnapshotException { + if (snapshotCount >= limit) { + String msg = "there are already " + snapshotCount + + " snapshot(s) and the " + type + " snapshot limit is " + + limit; + if (isImageLoaded()) { + // We have reached the maximum snapshot limit + throw new SnapshotException( + "Failed to create snapshot: " + msg); + } else { + // image is getting loaded. LOG an error msg and continue + LOG.error(msg); + } + } + } + + boolean isImageLoaded() { + return fsdir.isImageLoaded(); + } /** * Delete a snapshot for a snapshottable directory * @param snapshotName Name of the snapshot to be deleted @@ -352,8 +505,41 @@ public String createSnapshot(final LeaseManager leaseManager, */ public void deleteSnapshot(final INodesInPath iip, final String snapshotName, INode.ReclaimContext reclaimContext, long now) throws IOException { - INodeDirectory srcRoot = getSnapshottableRoot(iip); - srcRoot.removeSnapshot(reclaimContext, snapshotName, now); + final INodeDirectory srcRoot = getSnapshottableRoot(iip); + if (isSnapshotDeletionOrdered()) { + final DirectorySnapshottableFeature snapshottable + = srcRoot.getDirectorySnapshottableFeature(); + final Snapshot snapshot = snapshottable.getSnapshotByName( + srcRoot, snapshotName); + + // Diffs must be not empty since a snapshot exists in the list + final int earliest = snapshottable.getDiffs().getFirst().getSnapshotId(); + if (snapshot.getId() != earliest) { + final XAttr snapshotXAttr = buildXAttr(); + final List xattrs = Lists.newArrayListWithCapacity(1); + xattrs.add(snapshotXAttr); + + // The snapshot to be deleted is just marked for deletion in the xAttr. + // Same snaphot delete call can happen multiple times until and unless + // the very 1st instance of a snapshot delete hides it/remove it from + // snapshot list. XAttrSetFlag.REPLACE needs to be set to here in order + // to address this. + + // XAttr will set on the snapshot root directory + // NOTE : This function is directly called while replaying the edit + // logs.While replaying the edit logs we need to mark the snapshot + // deleted in the xattr of the snapshot root. + FSDirXAttrOp.unprotectedSetXAttrs(fsdir, + INodesInPath.append(iip, snapshot.getRoot(), + DFSUtil.string2Bytes(snapshotName)), xattrs, + EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); + return; + } + + assertFirstSnapshot(srcRoot, snapshottable, snapshot); + } + + srcRoot.removeSnapshot(reclaimContext, snapshotName, now, this); numSnapshots.getAndDecrement(); } @@ -397,9 +583,8 @@ void setSnapshotCounter(int counter) { snapshotCounter = counter; } - INodeDirectory[] getSnapshottableDirs() { - return snapshottables.values().toArray( - new INodeDirectory[snapshottables.size()]); + List getSnapshottableDirs() { + return new ArrayList<>(snapshottables.values()); } /** @@ -471,7 +656,37 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing( return statusList.toArray( new SnapshottableDirectoryStatus[statusList.size()]); } - + + /** + * List all the snapshots under a snapshottable directory. + */ + public SnapshotStatus[] getSnapshotListing(INodesInPath iip) + throws IOException { + INodeDirectory srcRoot = getSnapshottableRoot(iip); + ReadOnlyList snapshotList = srcRoot. + getDirectorySnapshottableFeature().getSnapshotList(); + SnapshotStatus[] statuses = new SnapshotStatus[snapshotList.size()]; + for (int count = 0; count < snapshotList.size(); count++) { + Snapshot s = snapshotList.get(count); + Snapshot.Root dir = s.getRoot(); + statuses[count] = new SnapshotStatus(dir.getModificationTime(), + dir.getAccessTime(), dir.getFsPermission(), + EnumSet.noneOf(HdfsFileStatus.Flags.class), + dir.getUserName(), dir.getGroupName(), + dir.getLocalNameBytes(), dir.getId(), + // the children number is same as the + // live fs as the children count is not cached per snashot. + // It is just used here to construct the HdfsFileStatus object. + // It is expensive to build the snapshot tree for the directory + // and determine the child count. + dir.getChildrenNum(Snapshot.CURRENT_STATE_ID), + s.getId(), s.getRoot().isMarkedAsDeleted(), + DFSUtil.string2Bytes(dir.getParent().getFullPathName())); + + } + return statuses; + } + /** * Compute the difference between two snapshots of a directory, or between a * snapshot of the directory and its current tree. @@ -483,7 +698,7 @@ public SnapshotDiffReport diff(final INodesInPath iip, // All the check for path has been included in the valueOf method. INodeDirectory snapshotRootDir; if (this.snapshotDiffAllowSnapRootDescendant) { - snapshotRootDir = getSnapshottableAncestorDir(iip); + snapshotRootDir = checkAndGetSnapshottableAncestorDir(iip); } else { snapshotRootDir = getSnapshottableRoot(iip); } @@ -516,7 +731,7 @@ public SnapshotDiffReportListing diff(final INodesInPath iip, // All the check for path has been included in the valueOf method. INodeDirectory snapshotRootDir; if (this.snapshotDiffAllowSnapRootDescendant) { - snapshotRootDir = getSnapshottableAncestorDir(iip); + snapshotRootDir = checkAndGetSnapshottableAncestorDir(iip); } else { snapshotRootDir = getSnapshottableRoot(iip); } @@ -545,6 +760,10 @@ public int getMaxSnapshotID() { return ((1 << SNAPSHOT_ID_BIT_WIDTH) - 1); } + public static XAttr buildXAttr() { + return XAttrHelper.buildXAttr(HdfsServerConstants.XATTR_SNAPSHOT_DELETED); + } + private ObjectName mxBeanName; public void registerMXBean() { @@ -584,15 +803,49 @@ public static SnapshottableDirectoryStatus.Bean toBean(INodeDirectory d) { d.getDirectorySnapshottableFeature().getNumSnapshots(), d.getDirectorySnapshottableFeature().getSnapshotQuota(), d.getModificationTime(), - Short.valueOf(Integer.toOctalString( - d.getFsPermissionShort())), + Short.parseShort(Integer.toOctalString(d.getFsPermissionShort())), d.getUserName(), d.getGroupName()); } public static SnapshotInfo.Bean toBean(Snapshot s) { + Snapshot.Root dir = s.getRoot(); return new SnapshotInfo.Bean( - s.getRoot().getLocalName(), s.getRoot().getFullPathName(), - s.getRoot().getModificationTime()); + s.getId(), + dir.getFullPathName(), + dir.getModificationTime(), + dir.isMarkedAsDeleted() + ); + } + + private List getSnapshottableDirsForGc() { + final List dirs = getSnapshottableDirs(); + Collections.shuffle(dirs); + return dirs; + } + + Snapshot.Root chooseDeletedSnapshot() { + for(INodeDirectory dir : getSnapshottableDirsForGc()) { + final Snapshot.Root root = chooseDeletedSnapshot(dir); + if (root != null) { + return root; + } + } + return null; + } + + private static Snapshot.Root chooseDeletedSnapshot(INodeDirectory dir) { + final DirectorySnapshottableFeature snapshottable + = dir.getDirectorySnapshottableFeature(); + if (snapshottable == null) { + return null; + } + final DirectoryWithSnapshotFeature.DirectoryDiffList diffs + = snapshottable.getDiffs(); + final Snapshot.Root first = (Snapshot.Root)diffs.getFirstSnapshotINode(); + if (first == null || !first.isMarkedAsDeleted()) { + return null; + } + return first; } -} +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementAttemptedItems.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementAttemptedItems.java index df4f0dddb49bb..ec583c710d533 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementAttemptedItems.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementAttemptedItems.java @@ -43,7 +43,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * A monitor class for checking whether block storage movements attempt diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java index b3f8de97b4f26..2bd52eb27fb9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java @@ -29,7 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * A Class to track the block collection IDs (Inode's ID) for which physical diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java index 4c04b466789be..77675479bd205 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java @@ -57,8 +57,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Setting storagePolicy on a file after the file write will only update the new diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java index 14cf05d4200d2..394ab12d443de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java @@ -31,7 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * This manages satisfy storage policy invoked path ids and expose methods to diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java index feba0e4f427e1..93eea6068c2fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java @@ -19,7 +19,7 @@ import java.net.InetAddress; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java index ba820323b20b1..e78e41957d107 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java @@ -19,11 +19,11 @@ import java.util.concurrent.TimeUnit; -import com.google.common.primitives.Ints; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * This class is a common place for NNTop configuration. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java index 72ec9f23446e8..0b6a18b46e3cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode.top.metrics; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java index 095294e811890..7ad4208717825 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java @@ -24,8 +24,8 @@ import java.util.Stack; import java.util.concurrent.ConcurrentHashMap; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/INodeCountVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/INodeCountVisitor.java new file mode 100644 index 0000000000000..a3c936547851d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/INodeCountVisitor.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.visitor; + +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * For validating {@link org.apache.hadoop.hdfs.server.namenode.FSImage}s. + */ +public class INodeCountVisitor implements NamespaceVisitor { + public interface Counts { + int getCount(INode inode); + } + + public static Counts countTree(INode root) { + return new INodeCountVisitor().count(root); + } + + private static class SetElement { + private final INode inode; + private final AtomicInteger count = new AtomicInteger(); + + SetElement(INode inode) { + this.inode = inode; + } + + int getCount() { + return count.get(); + } + + int incrementAndGet() { + return count.incrementAndGet(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } else if (obj == null || getClass() != obj.getClass()) { + return false; + } + final SetElement that = (SetElement) obj; + return this.inode.getId() == that.inode.getId(); + } + + @Override + public int hashCode() { + return Long.hashCode(inode.getId()); + } + } + + static class INodeSet implements Counts { + private final ConcurrentMap map + = new ConcurrentHashMap<>(); + + int put(INode inode, int snapshot) { + final SetElement key = new SetElement(inode); + final SetElement previous = map.putIfAbsent(key, key); + final SetElement current = previous != null? previous: key; + return current.incrementAndGet(); + } + + @Override + public int getCount(INode inode) { + final SetElement key = new SetElement(inode); + final SetElement value = map.get(key); + return value != null? value.getCount(): 0; + } + } + + private final INodeSet inodes = new INodeSet(); + + @Override + public INodeVisitor getDefaultVisitor() { + return new INodeVisitor() { + @Override + public void visit(INode iNode, int snapshot) { + inodes.put(iNode, snapshot); + } + }; + } + + private Counts count(INode root) { + root.accept(this, Snapshot.CURRENT_STATE_ID); + return inodes; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java new file mode 100644 index 0000000000000..ad7a6d9b6b68d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.visitor; + +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; +import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.INodeReference; +import org.apache.hadoop.hdfs.server.namenode.INodeSymlink; +import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature; +import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; +import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff; +import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature; +import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; + +import java.io.PrintWriter; +import java.io.StringWriter; + +/** + * To print the namespace tree recursively for testing. + * + * \- foo (INodeDirectory@33dd2717) + * \- sub1 (INodeDirectory@442172) + * +- file1 (INodeFile@78392d4) + * +- file2 (INodeFile@78392d5) + * +- sub11 (INodeDirectory@8400cff) + * \- file3 (INodeFile@78392d6) + * \- z_file4 (INodeFile@45848712) + */ +public final class NamespacePrintVisitor implements NamespaceVisitor { + static final String NON_LAST_ITEM = "+-"; + static final String LAST_ITEM = "\\-"; + + /** @return string of the tree in the given {@link FSNamesystem}. */ + public static String print2Sting(FSNamesystem ns) { + return print2Sting(ns.getFSDirectory().getRoot()); + } + + /** @return string of the tree from the given root. */ + public static String print2Sting(INode root) { + final StringWriter out = new StringWriter(); + new NamespacePrintVisitor(new PrintWriter(out)).print(root); + return out.getBuffer().toString(); + } + + private final PrintWriter out; + private final StringBuffer prefix = new StringBuffer(); + + private NamespacePrintVisitor(PrintWriter out) { + this.out = out; + } + + private void print(INode root) { + root.accept(this, Snapshot.CURRENT_STATE_ID); + } + + private void printINode(INode iNode, int snapshot) { + out.print(prefix); + out.print(" "); + final String name = iNode.getLocalName(); + out.print(name != null && name.isEmpty()? "/": name); + out.print(" ("); + out.print(iNode.getObjectString()); + out.print("), "); + out.print(iNode.getParentString()); + out.print(", " + iNode.getPermissionStatus(snapshot)); + } + + @Override + public void visitFile(INodeFile file, int snapshot) { + printINode(file, snapshot); + + out.print(", fileSize=" + file.computeFileSize(snapshot)); + // print only the first block, if it exists + out.print(", blocks="); + final BlockInfo[] blocks = file.getBlocks(); + out.print(blocks.length == 0 ? null: blocks[0]); + out.println(); + + final FileWithSnapshotFeature snapshotFeature + = file.getFileWithSnapshotFeature(); + if (snapshotFeature != null) { + if (prefix.length() >= 2) { + prefix.setLength(prefix.length() - 2); + prefix.append(" "); + } + out.print(prefix); + out.print(snapshotFeature); + } + out.println(); + } + + @Override + public void visitSymlink(INodeSymlink symlink, int snapshot) { + printINode(symlink, snapshot); + out.print(" ~> "); + out.println(symlink.getSymlinkString()); + } + + @Override + public void visitReference(INodeReference ref, int snapshot) { + printINode(ref, snapshot); + + if (ref instanceof INodeReference.DstReference) { + out.print(", dstSnapshotId=" + ref.getDstSnapshotId()); + } else if (ref instanceof INodeReference.WithCount) { + out.print(", " + ((INodeReference.WithCount)ref).getCountDetails()); + } + out.println(); + } + + @Override + public void preVisitReferred(INode referred) { + prefix.setLength(prefix.length() - 2); + prefix.append(" ->"); + } + + @Override + public void postVisitReferred(INode referred) { + prefix.setLength(prefix.length() - 2); + } + + @Override + public void visitDirectory(INodeDirectory dir, int snapshot) { + printINode(dir, snapshot); + + out.print(", childrenSize=" + dir.getChildrenList(snapshot).size()); + final DirectoryWithQuotaFeature q = dir.getDirectoryWithQuotaFeature(); + if (q != null) { + out.print(", " + q); + } + if (dir instanceof Snapshot.Root) { + out.print(", snapshotId=" + snapshot); + } + out.println(); + + if (prefix.length() >= 2) { + prefix.setLength(prefix.length() - 2); + prefix.append(" "); + } + + final DirectoryWithSnapshotFeature snapshotFeature + = dir.getDirectoryWithSnapshotFeature(); + if (snapshotFeature != null) { + out.print(prefix); + out.print(snapshotFeature); + } + out.println(); + } + + @Override + public void visitSnapshottable(INodeDirectory dir, + DirectorySnapshottableFeature snapshottable) { + out.println(); + out.print(prefix); + + out.print("Snapshot of "); + final String name = dir.getLocalName(); + out.print(name != null && name.isEmpty()? "/": name); + out.print(": quota="); + out.print(snapshottable.getSnapshotQuota()); + + int n = 0; + for(DirectoryDiff diff : snapshottable.getDiffs()) { + if (diff.isSnapshotRoot()) { + n++; + } + } + final int numSnapshots = snapshottable.getNumSnapshots(); + Preconditions.checkState(n == numSnapshots, + "numSnapshots = " + numSnapshots + " != " + n); + out.print(", #snapshot="); + out.println(n); + } + + @Override + public void preVisitSub(Element sub, int index, boolean isLast) { + prefix.append(isLast? LAST_ITEM : NON_LAST_ITEM); + } + + @Override + public void postVisitSub(Element sub, int index, boolean isLast) { + prefix.setLength(prefix.length() - 2); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespaceVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespaceVisitor.java new file mode 100644 index 0000000000000..2360b59373b22 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespaceVisitor.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.visitor; + +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; +import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.INodeReference; +import org.apache.hadoop.hdfs.server.namenode.INodeSymlink; +import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature; +import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; +import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; + +import java.util.Iterator; + +/** + * For visiting namespace trees. + */ +public interface NamespaceVisitor { + /** For visiting any {@link INode}. */ + interface INodeVisitor { + INodeVisitor DEFAULT = new INodeVisitor() {}; + + /** Visiting the given {@link INode}. */ + default void visit(INode iNode, int snapshot) { + } + } + + /** @return the default (non-recursive) {@link INodeVisitor}. */ + default INodeVisitor getDefaultVisitor() { + return INodeVisitor.DEFAULT; + } + + /** Visiting the given {@link INodeFile}. */ + default void visitFile(INodeFile file, int snapshot) { + getDefaultVisitor().visit(file, snapshot); + } + + /** Visiting the given {@link INodeSymlink}. */ + default void visitSymlink(INodeSymlink symlink, int snapshot) { + getDefaultVisitor().visit(symlink, snapshot); + } + + /** Visiting the given {@link INodeReference} (non-recursively). */ + default void visitReference(INodeReference ref, int snapshot) { + getDefaultVisitor().visit(ref, snapshot); + } + + /** First visit the given {@link INodeReference} and then the referred. */ + default void visitReferenceRecursively(INodeReference ref, int snapshot) { + visitReference(ref, snapshot); + + final INode referred = ref.getReferredINode(); + preVisitReferred(referred); + referred.accept(this, snapshot); + postVisitReferred(referred); + } + + /** Right before visiting the given referred {@link INode}. */ + default void preVisitReferred(INode referred) { + } + + /** Right after visiting the given referred {@link INode}. */ + default void postVisitReferred(INode referred) { + } + + /** Visiting the given {@link INodeDirectory} (non-recursively). */ + default void visitDirectory(INodeDirectory dir, int snapshot) { + getDefaultVisitor().visit(dir, snapshot); + } + + /** + * First visit the given {@link INodeDirectory}; + * then the children; + * and then, if snapshottable, the snapshots. */ + default void visitDirectoryRecursively(INodeDirectory dir, int snapshot) { + visitDirectory(dir, snapshot); + visitSubs(getChildren(dir, snapshot)); + + if (snapshot == Snapshot.CURRENT_STATE_ID) { + final DirectorySnapshottableFeature snapshottable + = dir.getDirectorySnapshottableFeature(); + if (snapshottable != null) { + visitSnapshottable(dir, snapshottable); + visitSubs(getSnapshots(snapshottable)); + } + } + } + + /** + * Right before visiting the given sub {@link Element}. + * The sub element may be a child of an {@link INodeDirectory} + * or a snapshot in {@link DirectorySnapshottableFeature}. + * + * @param sub the element to be visited. + * @param index the index of the sub element. + * @param isLast is the sub element the last element? + */ + default void preVisitSub(Element sub, int index, boolean isLast) { + } + + /** + * Right after visiting the given sub {@link Element}. + * The sub element may be a child of an {@link INodeDirectory} + * or a snapshot in {@link DirectorySnapshottableFeature}. + * + * @param sub the element just visited. + * @param index the index of the sub element. + * @param isLast is the sub element the last element? + */ + default void postVisitSub(Element sub, int index, boolean isLast) { + } + + /** Visiting a {@link DirectorySnapshottableFeature}. */ + default void visitSnapshottable(INodeDirectory dir, + DirectorySnapshottableFeature snapshottable) { + } + + /** + * Visiting the sub {@link Element}s recursively. + * + * @param subs the children of an {@link INodeDirectory} + * or the snapshots in {@link DirectorySnapshottableFeature}. + */ + default void visitSubs(Iterable subs) { + if (subs == null) { + return; + } + int index = 0; + for(final Iterator i = subs.iterator(); i.hasNext();) { + final Element e = i.next(); + final boolean isList = !i.hasNext(); + preVisitSub(e, index, isList); + e.getInode().accept(this, e.getSnapshotId()); + postVisitSub(e, index, isList); + index++; + } + } + + /** @return the children as {@link Element}s. */ + static Iterable getChildren(INodeDirectory dir, int snapshot) { + final Iterator i = dir.getChildrenList(snapshot).iterator(); + return new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + @Override + public boolean hasNext() { + return i.hasNext(); + } + + @Override + public Element next() { + return new Element(snapshot, i.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + }; + } + + /** @return the snapshots as {@link Element}s. */ + static Iterable getSnapshots( + DirectorySnapshottableFeature snapshottable) { + final Iterator i + = snapshottable.getDiffs().iterator(); + return new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + private DirectoryWithSnapshotFeature.DirectoryDiff next = findNext(); + + private DirectoryWithSnapshotFeature.DirectoryDiff findNext() { + for(; i.hasNext();) { + final DirectoryWithSnapshotFeature.DirectoryDiff diff = i.next(); + if (diff.isSnapshotRoot()) { + return diff; + } + } + return null; + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public Element next() { + final int id = next.getSnapshotId(); + final Element e = new Element(id, + snapshottable.getSnapshotById(id).getRoot()); + next = findNext(); + return e; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + }; + } + + /** Snapshot and INode. */ + class Element { + private final int snapshotId; + private final INode inode; + + Element(int snapshot, INode inode) { + this.snapshotId = snapshot; + this.inode = inode; + } + + INode getInode() { + return inode; + } + + int getSnapshotId() { + return snapshotId; + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/package.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/package.html new file mode 100644 index 0000000000000..f25748136d6f4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/package.html @@ -0,0 +1,21 @@ + + + +

Use the visitor pattern to visit namespace tree.

+ + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 2423a037c8fd0..e7bf32fcc05f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -86,6 +86,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; @@ -113,9 +114,9 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.StringUtils; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Charsets; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import com.sun.jersey.spi.container.ResourceFilters; /** Web-hdfs NameNode implementation. */ @@ -1340,24 +1341,66 @@ protected Response get( final String js = JsonUtil.toJsonString(snapshottableDirectoryList); return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } + case GETSNAPSHOTLIST: { + SnapshotStatus[] snapshotList = + cp.getSnapshotListing(fullpath); + final String js = JsonUtil.toJsonString(snapshotList); + return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + } default: throw new UnsupportedOperationException(op + " is not supported"); } } + /** + * Get the snapshot root of a given file or directory if it exists. + * e.g. if /snapdir1 is a snapshottable directory and path given is + * /snapdir1/path/to/file, this method would return /snapdir1 + * @param pathStr String of path to a file or a directory. + * @return Not null if found in a snapshot root directory. + * @throws IOException + */ + String getSnapshotRoot(String pathStr) throws IOException { + SnapshottableDirectoryStatus[] dirStatusList = + getRpcClientProtocol().getSnapshottableDirListing(); + if (dirStatusList == null) { + return null; + } + for (SnapshottableDirectoryStatus dirStatus : dirStatusList) { + String currDir = dirStatus.getFullPath().toString(); + if (pathStr.startsWith(currDir)) { + return currDir; + } + } + return null; + } + private String getTrashRoot(Configuration conf, String fullPath) throws IOException { - UserGroupInformation ugi= UserGroupInformation.getCurrentUser(); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); String parentSrc = getParent(fullPath); + String ssTrashRoot = ""; + boolean isSnapshotTrashRootEnabled = getRpcClientProtocol() + .getServerDefaults().getSnapshotTrashRootEnabled(); + if (isSnapshotTrashRootEnabled) { + String ssRoot = getSnapshotRoot(fullPath); + if (ssRoot != null) { + ssTrashRoot = DFSUtilClient.getSnapshotTrashRoot(ssRoot, ugi); + } + } EncryptionZone ez = getRpcClientProtocol().getEZForPath( parentSrc != null ? parentSrc : fullPath); - String trashRoot; + String ezTrashRoot = ""; if (ez != null) { - trashRoot = DFSUtilClient.getEZTrashRoot(ez, ugi); + ezTrashRoot = DFSUtilClient.getEZTrashRoot(ez, ugi); + } + // Choose the longest path + if (ssTrashRoot.isEmpty() && ezTrashRoot.isEmpty()) { + return DFSUtilClient.getTrashRoot(conf, ugi); } else { - trashRoot = DFSUtilClient.getTrashRoot(conf, ugi); + return ssTrashRoot.length() > ezTrashRoot.length() ? + ssTrashRoot : ezTrashRoot; } - return trashRoot; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECReconstructionCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECReconstructionCommand.java index 6e9c55be2c24b..b2495c8d6d9d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECReconstructionCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECReconstructionCommand.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.protocol; -import com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java index 926cce91be9a3..b2a11f0a171fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; /** * BlockRecoveryCommand is an instruction to a data-node to recover diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java index b6eddb61579da..f60d748dc9f56 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.protocol; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.StorageType; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java index 711973838bc06..d7c2466e2eeee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.StorageInfo; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * DatanodeRegistration class contains all information the name-node needs diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java index 10650da75d14a..81a7e457891cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java @@ -30,8 +30,8 @@ import org.apache.hadoop.hdfs.server.namenode.NNStorage; import org.apache.hadoop.util.VersionInfo; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * NamespaceInfo is returned by the name-node in reply diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java index 427daf132962f..8c27f30978324 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.protocol; -import com.google.common.collect.ComparisonChain; +import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; import java.util.function.Function; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java index 8252b3bc7f702..391078f558509 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java @@ -20,8 +20,8 @@ import java.util.Collections; import java.util.List; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java index 27cdf70279087..f619fb40f55d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java @@ -18,7 +18,7 @@ */ package org.apache.hadoop.hdfs.tools; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -43,7 +43,7 @@ public class AdminHelper { static final int MAX_LINE_WIDTH = 80; static final String HELP_COMMAND_NAME = "-help"; - static DistributedFileSystem getDFS(Configuration conf) + public static DistributedFileSystem getDFS(Configuration conf) throws IOException { FileSystem fs = FileSystem.get(conf); return checkAndGetDFS(fs, conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java index 9e7a3cb9c753b..a22d34bf007e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java @@ -44,7 +44,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; -import com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import org.apache.hadoop.util.ToolRunner; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index ec5fa0afddc60..f8ff5fe99392a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -38,8 +38,9 @@ import java.util.TreeSet; import java.util.concurrent.TimeUnit; -import com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.hdfs.client.HdfsAdmin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -103,7 +104,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * This class provides some DFS administrative access shell commands. @@ -239,7 +240,8 @@ private static class ClearSpaceQuotaCommand extends DFSAdminCommand { "\t\t- DISK\n" + "\t\t- SSD\n" + "\t\t- ARCHIVE\n" + - "\t\t- PROVIDED"; + "\t\t- PROVIDED\n" + + "\t\t- NVDIMM"; private StorageType type; @@ -303,7 +305,8 @@ private static class SetSpaceQuotaCommand extends DFSAdminCommand { "\t\t- DISK\n" + "\t\t- SSD\n" + "\t\t- ARCHIVE\n" + - "\t\t- PROVIDED"; + "\t\t- PROVIDED\n" + + "\t\t- NVDIMM"; private long quota; // the quota to be set private StorageType type; @@ -458,6 +461,7 @@ static int run(DistributedFileSystem dfs, String[] argv, int idx) throws IOExcep "\t[-fetchImage ]\n" + "\t[-allowSnapshot ]\n" + "\t[-disallowSnapshot ]\n" + + "\t[-provisionSnapshotTrash ]\n" + "\t[-shutdownDatanode [upgrade]]\n" + "\t[-evictWriters ]\n" + "\t[-getDatanodeInfo ]\n" + @@ -763,9 +767,9 @@ public int triggerBlockReport(String[] argv) throws IOException { */ public void allowSnapshot(String[] argv) throws IOException { Path p = new Path(argv[1]); - final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), getConf()); + final HdfsAdmin admin = new HdfsAdmin(p.toUri(), getConf()); try { - dfs.allowSnapshot(p); + admin.allowSnapshot(p); } catch (SnapshotException e) { throw new RemoteException(e.getClass().getName(), e.getMessage()); } @@ -780,14 +784,33 @@ public void allowSnapshot(String[] argv) throws IOException { */ public void disallowSnapshot(String[] argv) throws IOException { Path p = new Path(argv[1]); - final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), getConf()); + final HdfsAdmin admin = new HdfsAdmin(p.toUri(), getConf()); try { - dfs.disallowSnapshot(p); + admin.disallowSnapshot(p); } catch (SnapshotException e) { throw new RemoteException(e.getClass().getName(), e.getMessage()); } System.out.println("Disallowing snapshot on " + argv[1] + " succeeded"); } + + /** + * Provision trash root in a snapshottable directory. + * Usage: hdfs dfsadmin -provisionSnapshotTrash snapshotDir + * @param argv List of of command line parameters. + * @exception IOException + */ + public void provisionSnapshotTrash(String[] argv) throws IOException { + Path p = new Path(argv[1]); + final HdfsAdmin admin = new HdfsAdmin(p.toUri(), getConf()); + Path trashRoot; + try { + trashRoot = admin.provisionSnapshotTrash(p); + } catch (SnapshotException e) { + throw new RemoteException(e.getClass().getName(), e.getMessage()); + } + System.out.println("Successfully provisioned snapshot trash at " + + trashRoot); + } /** * Command to ask the namenode to save the namespace. @@ -1243,6 +1266,10 @@ private void printHelp(String cmd) { String disallowSnapshot = "-disallowSnapshot :\n" + "\tDo not allow snapshots to be taken on a directory any more.\n"; + String provisionSnapshotTrash = "-provisionSnapshotTrash :\n" + + "\tProvision trash root in a snapshottable directory with permission" + + "\t" + HdfsAdmin.TRASH_PERMISSION + ".\n"; + String shutdownDatanode = "-shutdownDatanode [upgrade]\n" + "\tSubmit a shutdown request for the given datanode. If an optional\n" + "\t\"upgrade\" argument is specified, clients accessing the datanode\n" @@ -1332,6 +1359,8 @@ private void printHelp(String cmd) { System.out.println(allowSnapshot); } else if ("disallowSnapshot".equalsIgnoreCase(cmd)) { System.out.println(disallowSnapshot); + } else if ("provisionSnapshotTrash".equalsIgnoreCase(cmd)) { + System.out.println(provisionSnapshotTrash); } else if ("shutdownDatanode".equalsIgnoreCase(cmd)) { System.out.println(shutdownDatanode); } else if ("evictWriters".equalsIgnoreCase(cmd)) { @@ -1374,6 +1403,7 @@ private void printHelp(String cmd) { System.out.println(fetchImage); System.out.println(allowSnapshot); System.out.println(disallowSnapshot); + System.out.println(provisionSnapshotTrash); System.out.println(shutdownDatanode); System.out.println(evictWriters); System.out.println(getDatanodeInfo); @@ -2083,6 +2113,9 @@ private static void printUsage(String cmd) { } else if ("-disallowSnapshot".equalsIgnoreCase(cmd)) { System.err.println("Usage: hdfs dfsadmin" + " [-disallowSnapshot ]"); + } else if ("-provisionSnapshotTrash".equalsIgnoreCase(cmd)) { + System.err.println("Usage: hdfs dfsadmin" + + " [-provisionSnapshotTrash ]"); } else if ("-saveNamespace".equals(cmd)) { System.err.println("Usage: hdfs dfsadmin" + " [-saveNamespace [-beforeShutdown]]"); @@ -2184,7 +2217,6 @@ private static void printUsage(String cmd) { /** * @param argv The parameters passed to this program. - * @exception Exception if the filesystem does not exist. * @return 0 on success, non zero on error. */ @Override @@ -2217,6 +2249,11 @@ public int run(String[] argv) { printUsage(cmd); return exitCode; } + } else if ("-provisionSnapshotTrash".equalsIgnoreCase(cmd)) { + if (argv.length != 2) { + printUsage(cmd); + return exitCode; + } } else if ("-report".equals(cmd)) { if (argv.length > 6) { printUsage(cmd); @@ -2353,6 +2390,8 @@ public int run(String[] argv) { allowSnapshot(argv); } else if ("-disallowSnapshot".equalsIgnoreCase(cmd)) { disallowSnapshot(argv); + } else if ("-provisionSnapshotTrash".equalsIgnoreCase(cmd)) { + provisionSnapshotTrash(argv); } else if ("-saveNamespace".equals(cmd)) { exitCode = saveNamespace(argv); } else if ("-rollEdits".equals(cmd)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java index db03db6e57c96..15c63732f7a69 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java @@ -23,9 +23,9 @@ import java.util.Collection; import java.util.Map; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSortedMap; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSortedMap; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.hadoop.ha.FailoverController; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java index 1736e42232507..4773beb85de08 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java @@ -29,7 +29,7 @@ import java.util.ArrayList; import java.util.List; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.HadoopIllegalArgumentException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java index 9a654e580cb06..89389a094686a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java @@ -33,7 +33,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java index 0dc4bcba4d24f..10156287be15d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java @@ -43,7 +43,7 @@ import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.GenericOptionsParser; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Fetch a DelegationToken from the current Namenode and store it in the diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java index c4527e59b2884..94aff53470b72 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java @@ -33,7 +33,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.net.NetUtils; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java index a6901f335d1fb..7238c58cb579b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java @@ -41,7 +41,7 @@ import org.xml.sax.helpers.DefaultHandler; import org.xml.sax.helpers.XMLReaderFactory; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; /** * OfflineEditsXmlLoader walks an EditsVisitor over an OEV XML file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java index cc97ea71d28e5..411df1062771b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java @@ -30,7 +30,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; /** * StatisticsEditsVisitor implements text version of EditsVisitor diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java index 9b0031a3a2eaa..f14ee5f930fe0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.tools.offlineImageViewer; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.ChannelFutureListener; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java index 7032e2a776d34..0d4781d12f7ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java @@ -30,7 +30,7 @@ import java.util.List; import java.util.Map; -import com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; @@ -57,9 +57,9 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.LimitInputStream; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; /** * FSImageLoader loads fsimage and provide methods to return JSON formatted diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java index 25a7bbe40bd0a..54b183b7b6965 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java @@ -33,7 +33,7 @@ import org.apache.hadoop.util.LimitInputStream; import org.apache.hadoop.util.StringUtils; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * This is the tool for analyzing file sizes in the namespace image. In order to diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java index 7eccf75dc65e5..9ad4b090649b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.tools.offlineImageViewer; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_NAME_MASK; import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_NAME_OFFSET; import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_SCOPE_OFFSET; @@ -49,8 +49,8 @@ import java.util.LinkedList; import java.util.Map; -import com.google.common.io.CountingOutputStream; -import com.google.common.primitives.Ints; +import org.apache.hadoop.thirdparty.com.google.common.io.CountingOutputStream; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.thirdparty.protobuf.TextFormat; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java index 7e5eb3177f95b..737e7384b9a7c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.tools.offlineImageViewer; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.server.namenode.FsImageProto; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java index 1bd7d97f8a155..cd4047d9e646c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hdfs.tools.offlineImageViewer; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringEscapeUtils; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java index 2c4a8946e8bd2..920dfdc6dc933 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java @@ -68,8 +68,8 @@ import org.apache.hadoop.hdfs.util.XMLUtils; import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.util.LimitInputStream; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.util.VersionInfo; import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_MASK; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java index 7f837fd32d6d9..f732bd6a7f542 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java @@ -22,7 +22,7 @@ import java.nio.file.Files; import java.nio.file.Paths; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; /** * TextWriterImageProcessor mixes in the ability for ImageVisitor diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/WebImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/WebImageViewer.java index 767ecd809e048..29ac759576c28 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/WebImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/WebImageViewer.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.tools.offlineImageViewer; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.Channel; import io.netty.channel.ChannelInitializer; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshot.java new file mode 100644 index 0000000000000..65b38afe416eb --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshot.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools.snapshot; + + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; +import org.apache.hadoop.hdfs.tools.AdminHelper; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +/** + * A tool used to list all snapshottable directories that are owned by the + * current user. The tool returns all the snapshottable directories if the user + * is a super user. + */ +@InterfaceAudience.Private +public class LsSnapshot extends Configured implements Tool { + @Override + public int run(String[] argv) throws Exception { + String description = "hdfs lsSnapshot : \n" + + "\tGet the list of snapshots for a snapshottable directory.\n"; + + if(argv.length != 1) { + System.err.println("Invalid no of arguments"); + System.err.println("Usage: \n" + description); + return 1; + } + Path snapshotRoot = new Path(argv[0]); + try { + DistributedFileSystem dfs = AdminHelper.getDFS(getConf()); + SnapshotStatus[] stats = dfs.getSnapshotListing(snapshotRoot); + SnapshotStatus.print(stats, System.out); + } catch (Exception e) { + String[] content = e.getLocalizedMessage().split("\n"); + System.err.println("lsSnapshot: " + content[0]); + return 1; + } + return 0; + } + public static void main(String[] argv) throws Exception { + int rc = ToolRunner.run(new LsSnapshot(), argv); + System.exit(rc); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/AtomicFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/AtomicFileOutputStream.java index 33f976a7f7ada..c6003e8b9a3b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/AtomicFileOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/AtomicFileOutputStream.java @@ -22,6 +22,7 @@ import java.io.FileOutputStream; import java.io.FilterOutputStream; import java.io.IOException; +import java.nio.file.Files; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,8 +76,12 @@ public void close() throws IOException { boolean renamed = tmpFile.renameTo(origFile); if (!renamed) { // On windows, renameTo does not replace. - if (origFile.exists() && !origFile.delete()) { - throw new IOException("Could not delete original file " + origFile); + if (origFile.exists()) { + try { + Files.delete(origFile.toPath()); + } catch (IOException e) { + throw new IOException("Could not delete original file " + origFile, e); + } } try { NativeIO.renameTo(tmpFile, origFile); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/BestEffortLongFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/BestEffortLongFile.java index 2fdf5472bee03..375630db6bdc8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/BestEffortLongFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/BestEffortLongFile.java @@ -28,8 +28,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.io.IOUtils; -import com.google.common.io.Files; -import com.google.common.primitives.Longs; +import org.apache.hadoop.thirdparty.com.google.common.io.Files; +import org.apache.hadoop.thirdparty.com.google.common.primitives.Longs; /** * Class that represents a file on disk which stores a single long diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java index 4323f2d4c5637..21a7bb58750b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.util; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import java.util.ArrayList; import java.util.Collections; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java index dded84800ec33..880bf6edb2516 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.util; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.commons.lang3.ArrayUtils; import java.util.Arrays; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumDoubles.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumDoubles.java index 126070aa016a1..fee687edf54b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumDoubles.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumDoubles.java @@ -19,7 +19,7 @@ import java.util.Arrays; -import com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; /** * Similar to {@link EnumCounters} except that the value type is double. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/MD5FileUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/MD5FileUtils.java index b541b14f64924..2bc63ec77eb29 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/MD5FileUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/MD5FileUtils.java @@ -35,7 +35,7 @@ import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.util.StringUtils; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; /** * Static functions for dealing with files of the same format diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java index a94d7edff08b3..78834e0161a4f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java @@ -28,7 +28,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.io.IOUtils; -import com.google.common.base.Charsets; +import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; /** * Class that represents a file on disk which persistently stores diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ReferenceCountMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ReferenceCountMap.java index 5b29c4308b8bb..f24d1aa9df915 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ReferenceCountMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ReferenceCountMap.java @@ -23,8 +23,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; /** * Class for de-duplication of instances.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index bf5568bc3e563..aa490ef940c75 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.web; -import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileChecksum; @@ -40,7 +40,7 @@ import org.apache.hadoop.util.StringUtils; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import java.io.IOException; import java.util.*; @@ -588,6 +588,17 @@ public static String toJsonString( return toJsonString("SnapshottableDirectoryList", a); } + public static String toJsonString(SnapshotStatus[] snapshotList) { + if (snapshotList == null) { + return toJsonString("SnapshotList", null); + } + Object[] a = new Object[snapshotList.length]; + for (int i = 0; i < snapshotList.length; i++) { + a[i] = toJsonMap(snapshotList[i]); + } + return toJsonString("SnapshotList", a); + } + private static Object toJsonMap( SnapshottableDirectoryStatus snapshottableDirectoryStatus) { final Map m = new TreeMap(); @@ -599,6 +610,19 @@ private static Object toJsonMap( return m; } + private static Object toJsonMap( + SnapshotStatus snapshotStatus) { + final Map m = new TreeMap(); + HdfsFileStatus status = snapshotStatus.getDirStatus(); + m.put("snapshotID", snapshotStatus.getSnapshotID()); + m.put("deletionStatus", snapshotStatus.isDeleted() ? "DELETED" : "ACTIVE"); + m.put("fullPath", SnapshotStatus.getSnapshotPath( + DFSUtilClient.bytes2String(snapshotStatus.getParentFullPath()), + status.getLocalName())); + m.put("dirStatus", toJsonMap(snapshotStatus.getDirStatus())); + return m; + } + private static Map toJsonMap( final BlockLocation blockLocation) throws IOException { if (blockLocation == null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ExceptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ExceptionHandler.java index 2e0a17efa4b30..02b70c16be649 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ExceptionHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ExceptionHandler.java @@ -35,7 +35,7 @@ import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.token.SecretManager.InvalidToken; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import com.sun.jersey.api.ParamException; import com.sun.jersey.api.container.ContainerException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 689ecfe2f3342..8d70c5737fa22 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -313,6 +313,17 @@ + + dfs.namenode.redundancy.considerLoadByStorageType + false + + Decide if chooseTarget considers the target's load with respect to the + storage type. Typically to be used when datanodes contain homogenous + storage types. Irrelevent if dfs.namenode.redundancy.considerLoad is + false. + + + dfs.namenode.redundancy.considerLoad.factor 2.0 @@ -377,7 +388,7 @@ 0 Reserved space in bytes per volume. Always leave this much space free for non dfs use. Specific storage type based reservation is also supported. The property can be followed with - corresponding storage types ([ssd]/[disk]/[archive]/[ram_disk]) for cluster with heterogeneous storage. + corresponding storage types ([ssd]/[disk]/[archive]/[ram_disk]/[nvdimm]) for cluster with heterogeneous storage. For example, reserved space for RAM_DISK storage can be configured using property 'dfs.datanode.du.reserved.ram_disk'. If specific storage type reservation is not configured then dfs.datanode.du.reserved will be used. Support multiple size unit suffix(case insensitive), @@ -395,7 +406,7 @@ when this takes effect. The actual number of bytes reserved will be calculated by using the total capacity of the data directory in question. Specific storage type based reservation is also supported. The property can be followed with corresponding storage types - ([ssd]/[disk]/[archive]/[ram_disk]) for cluster with heterogeneous storage. + ([ssd]/[disk]/[archive]/[ram_disk]/[nvdimm]) for cluster with heterogeneous storage. For example, reserved percentage space for RAM_DISK storage can be configured using property 'dfs.datanode.du.reserved.pct.ram_disk'. If specific storage type reservation is not configured then dfs.datanode.du.reserved.pct will be used. @@ -604,7 +615,7 @@ should store its blocks. If this is a comma-delimited list of directories, then data will be stored in all named directories, typically on different devices. The directories should be tagged - with corresponding storage types ([SSD]/[DISK]/[ARCHIVE]/[RAM_DISK]) for HDFS + with corresponding storage types ([SSD]/[DISK]/[ARCHIVE]/[RAM_DISK]/[NVDIMM]) for HDFS storage policies. The default storage type will be DISK if the directory does not have a storage type tagged explicitly. Directories that do not exist will be created if local filesystem permission allows. @@ -1236,6 +1247,14 @@ restarts. Defaults to 10 minutes. + + dfs.client.write.recover.lease.on.close.exception + false + + Set to true to call recoverLease operation automatically when DFSOutputSteam closing encounters exception. + + + dfs.namenode.checkpoint.dir file://${hadoop.tmp.dir}/dfs/namesecondary @@ -1428,13 +1447,13 @@ dfs.image.transfer.bandwidthPerSec - 0 + 52428800 Maximum bandwidth used for regular image transfers (instead of bootstrapping the standby namenode), in bytes per second. This can help keep normal namenode operations responsive during checkpointing. - A default value of 0 indicates that throttling is disabled. + A default value is 50mb per second. The maximum bandwidth used for bootstrapping standby namenode is configured with dfs.image.transfer-bootstrap-standby.bandwidthPerSec. Support multiple size unit suffix(case insensitive), as described @@ -1596,6 +1615,15 @@ + + dfs.block.scanner.volume.join.timeout.ms + 5000 + + The amount of time in milliseconds that the BlockScanner times out waiting + for the VolumeScanner thread to join during a shutdown call. + + + dfs.datanode.readahead.bytes 4194304 @@ -3288,7 +3316,7 @@ dfs.datanode.block.id.layout.upgrade.threads - 12 + 6 The number of threads to use when creating hard links from current to previous blocks during upgrade of a DataNode to block ID-based block layout (see HDFS-6482 for details on the layout). @@ -4185,6 +4213,15 @@ + + dfs.client.read.uri.cache.enabled + false + + If true, dfs client will use cache when creating URI based on host:port + to reduce the frequency of URI object creation. + + + dfs.client.read.short.circuit.replica.stale.threshold.ms 1800000 @@ -4421,6 +4458,14 @@ + + dfs.data.transfer.max.packet.size + 16777216 + + The max size of any single packet. + + + dfs.datanode.balance.max.concurrent.moves 100 @@ -5101,6 +5146,15 @@ for maximum no of snapshots allowed is 65536. + + dfs.namenode.snapshot.filesystem.limit + 65536 + + Limits the maximum number of snapshots allowed on the entire filesystem. + If the configuration is not set, the default limit + for maximum no of snapshots allowed is 65536. + + dfs.namenode.snapshot.skiplist.max.levels @@ -5966,4 +6020,29 @@ + + dfs.datanode.same-disk-tiering.enabled + false + + HDFS-15548 to allow DISK/ARCHIVE to be + configured on the same disk mount to manage disk IO. + When this is enabled, datanode will control the capacity + of DISK/ARCHIVE based on reserve-for-archive.percentage. + + + + + dfs.datanode.reserve-for-archive.default.percentage + 0.0 + + Default disk capacity ratio of ARCHIVE volume, + expected the value to be between 0 to 1. + This will be applied when DISK/ARCHIVE volumes are configured + on the same mount, which is detected by datanode. + Beware that capacity usage might be >100% if there are already + data blocks exist and the configured ratio is small, which will + prevent the volume from taking new blocks + until capacity is balanced out. + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html index 39680e84a8c82..7301064651e2b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html @@ -125,7 +125,7 @@ {/dn.VolumeInfo} - + @@ -134,4 +134,4 @@ - \ No newline at end of file + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index fcf4e0d058613..bb3cf4a475ca8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -250,13 +250,15 @@ - - - + @@ -105,4 +105,4 @@ - \ No newline at end of file + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html index e9c206eda0381..41a468d4cfd80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html @@ -86,7 +86,7 @@ {/snn} -