Merge

.asf.yaml

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+github:
+  enabled_merge_buttons:
+    squash: true
+    merge: false
+    rebase: false
+notifications:
+  commits:      common-commits@hadoop.apache.org
+  issues:       common-issues@hadoop.apache.org
+  pullrequests: common-issues@hadoop.apache.org
+  jira_options: link label worklog

.gitignore

@@ -51,6 +51,8 @@ patchprocess/
 .history/
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log
+hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/schema
+phantomjsdriver.log
 
 #robotframework outputs
 log.html

BUILDING.txt

@@ -82,7 +82,7 @@ Installing required packages for clean install of Ubuntu 14.04 LTS Desktop:
 
 Optional packages:
 
-* Snappy compression
+* Snappy compression (only used for hadoop-mapreduce-client-nativetask)
   $ sudo apt-get install snappy libsnappy-dev
 * Intel ISA-L library for erasure coding
   Please refer to https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
@@ -168,7 +168,8 @@ Maven build goals:
 
    Snappy is a compression library that can be utilized by the native code.
    It is currently an optional component, meaning that Hadoop can be built with
-   or without this dependency.
+   or without this dependency. Snappy library as optional dependency is only
+   used for hadoop-mapreduce-client-nativetask.
 
   * Use -Drequire.snappy to fail the build if libsnappy.so is not found.
     If this option is not specified and the snappy library is missing,
@@ -436,6 +437,9 @@ Building on CentOS 8
   $ sudo dnf group install --with-optional 'Development Tools'
   $ sudo dnf install java-1.8.0-openjdk-devel maven
 
+* Install python2 for building documentation.
+  $ sudo dnf install python2
+
 * Install Protocol Buffers v3.7.1.
   $ git clone https://github.com/protocolbuffers/protobuf
   $ cd protobuf
@@ -449,8 +453,16 @@ Building on CentOS 8
 * Install libraries provided by CentOS 8.
   $ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel
 
+* Install boost.
+  $ curl -L -o boost_1_72_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download
+  $ tar xjf boost_1_72_0.tar.bz2
+  $ cd boost_1_72_0
+  $ ./bootstrap.sh --prefix=/usr/local
+  $ ./b2
+  $ sudo ./b2 install
+
 * Install optional dependencies (snappy-devel).
-  $ sudo dnf --enablerepo=PowerTools snappy-devel
+  $ sudo dnf --enablerepo=PowerTools install snappy-devel
 
 * Install optional dependencies (libzstd-devel).
   $ sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm

Jenkinsfile

@@ -35,7 +35,7 @@ pipeline {
         DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
         YETUS='yetus'
         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
-        YETUS_VERSION='rel/0.12.0'
+        YETUS_VERSION='6ab19e71eaf3234863424c6f684b34c1d3dcc0ce'
     }
 
     parameters {
@@ -96,8 +96,8 @@ pipeline {
                         YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
 
                         # our project defaults come from a personality file
-                        # which will get loaded automatically by setting the project name
                         YETUS_ARGS+=("--project=hadoop")
+                        YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh")
 
                         # lots of different output formats
                         YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
@@ -144,6 +144,7 @@ pipeline {
                         # Dockerfile since we don't want to use the auto-pulled version.
                         YETUS_ARGS+=("--docker")
                         YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
+                        YETUS_ARGS+=("--mvn-custom-repos")
 
                         # effectively treat dev-suport as a custom maven module
                         YETUS_ARGS+=("--skip-dirs=dev-support")
@@ -159,6 +160,9 @@ pipeline {
                         YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
                         YETUS_ARGS+=("--multijdktests=compile")
 
+                        # custom javadoc goals
+                        YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork")
+
                         "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
                         '''
                 }

LICENSE-binary

@@ -405,7 +405,7 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.4.1.min.js
+hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js
 hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js

LICENSE.txt

@@ -245,7 +245,7 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.4.1.min.js
+hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js
 hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js

dev-support/bin/dist-copynativelibs

@@ -111,9 +111,6 @@ for i in "$@"; do
     --openssllibbundle=*)
       OPENSSLLIBBUNDLE=${i#*=}
     ;;
-    --snappybinbundle=*)
-      SNAPPYBINBUNDLE=${i#*=}
-    ;;
     --snappylib=*)
       SNAPPYLIB=${i#*=}
     ;;
@@ -176,8 +173,6 @@ if [[ -d "${BIN_DIR}" ]] ; then
     exit 1
   fi
 
-  bundle_native_bin "${SNAPPYBINBUNDLE}" "${SNAPPYLIBBUNDLE}" "snappy.lib" "snappy" "${SNAPPYLIB}"
-
   bundle_native_bin "${ZSTDBINBUNDLE}" "${ZSTDLIBBUNDLE}" "zstd.lib" "zstd" "${ZSTDLIB}"
 
   bundle_native_bin "${OPENSSLBINBUNDLE}" "${OPENSSLLIBBUNDLE}" "openssl.lib" "crypto" "${OPENSSLLIB}"