IMPALA-4669: [KUTIL] Add kudu_util library to the build.

NOTE: This commit is part of a set of changes for IMPALA-7006. It
contains pieces of a previous commit that need to be cherry picked
again after rebasing the code in be/src/kudu/{util,security,rpc}.

The original commit message is below:

A few miscellaneous changes to allow kudu_util to compile with Impala.

Add kudu_version.cc to substitute for the version.cc file that is
automatically built during the full Kudu build.

Set LZ4_DISABLE_DEPRECATE_WARNINGS to allow Kudu's compressor utility to
use deprecated names for LZ4 methods.

Add NO_NVM_SUPPORT flag to Kudu build (plan to upstream this later) to
disable building with nvm support, removing a library dependency.

Also remove imported FindOpenSSL.cmake in favour of the standard one provided
by cmake itself.

Finally, a few changes to allow compilation on RHEL5:

* Only use sched_getcpu() if supported
* Only include magic.h if available
* Workaround for kernels that don't have SOCK_NONBLOCK
* Workaround for kernels that don't have O_CLOEXEC (ignore the flag)
* Provide non-working implementation of fallocate()
* Disable inclusion of linux/fiemap.h - although this exists on RHEL5,
  it does not compile due to other #includes in env_posix.cc. We disable
  the path this is used for, since Impala does not call that code.
* Use Kudu's implementation of pipe(2), preadv(2) and pwritev(2) where
  it doesn't exist.

In most cases these changes simply force kutil to revert to a different
implementation that was already written for OSX support - this patch
generalises the logic to provide the implementation whenever the
required function doesn't exist.

This patch compiles on RHEL5.5 and 6.0, SLES11 and 12, Ubuntu 12.04 and
14.04 and Debian 7.0 and 8.0.

Change-Id: I451f02d3e4669e8a548b92fb1445cb2b322659a2
Reviewed-on: http://gerrit.cloudera.org:8080/5715
Tested-by: Impala Public Jenkins
Reviewed-by: Henry Robinson <henry@cloudera.com>
Reviewed-on: http://gerrit.cloudera.org:8080/10758
Reviewed-by: Michael Ho <kwho@cloudera.com>
Tested-by: Lars Volker <lv@cloudera.com>

CMakeLists.txt

@@ -19,8 +19,6 @@ cmake_minimum_required(VERSION 3.2.3)
 
 # This is a Kudu-specific flag that disables Kudu targets that are test-only.
 set(NO_TESTS 1)
-# Kudu-specific, disable NVM support so that libvmem is not a dependency.
-set(NO_NVM_SUPPORT 1)
 
 # Explicitly define project() to allow modifying the compiler before the project is
 # initialized.
@@ -348,7 +346,8 @@ endif()
 if (NOT APPLE)
   find_package(LibUnwind REQUIRED)
   include_directories(SYSTEM ${LIBUNWIND_INCLUDE_DIR})
-  IMPALA_ADD_THIRDPARTY_LIB(libunwind ${LIBUNWIND_INCLUDE_DIR} ${LIBUNWIND_STATIC_LIB} "")
+  IMPALA_ADD_THIRDPARTY_LIB(libunwind ${LIBUNWIND_INCLUDE_DIR} ${LIBUNWIND_STATIC_LIB}
+    ${LIBUNWIND_SHARED_LIB})
 endif()
 
 # Required for KRPC_GENERATE, which converts protobuf to stubs.

LICENSE.txt

@@ -798,6 +798,67 @@ Some portions of this module are derived from code from krb5
 
 --------------------------------------------------------------------------------
 
+be/src/kudu/util/array_view.h: 3-clause BSD license with patent grant
+
+  Copyright (c) 2015, The WebRTC project authors. All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are
+  met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+
+    * Neither the name of Google nor the names of its contributors may
+      be used to endorse or promote products derived from this software
+      without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+  Additional IP Rights Grant (Patents)
+  ------------------------------------
+
+  "This implementation" means the copyrightable works distributed by
+  Google as part of the WebRTC code package.
+
+  Google hereby grants to you a perpetual, worldwide, non-exclusive,
+  no-charge, irrevocable (except as stated in this section) patent
+  license to make, have made, use, offer to sell, sell, import,
+  transfer, and otherwise run, modify and propagate the contents of this
+  implementation of the WebRTC code package, where such license applies
+  only to those patent claims, both currently owned by Google and
+  acquired in the future, licensable by Google that are necessarily
+  infringed by this implementation of the WebRTC code package. This
+  grant does not include claims that would be infringed only as a
+  consequence of further modification of this implementation. If you or
+  your agent or exclusive licensee institute or order or agree to the
+  institution of patent litigation against any entity (including a
+  cross-claim or counterclaim in a lawsuit) alleging that this
+  implementation of the WebRTC code package or any code incorporated
+  within this implementation of the WebRTC code package constitutes
+  direct or contributory patent infringement, or inducement of patent
+  infringement, then any patent rights granted to you under this License
+  for this implementation of the WebRTC code package shall terminate as
+  of the date such litigation is filed.
+
+--------------------------------------------------------------------------------
+
 be/src/kudu/util/x509_check_host.*: OpenSSL software license:
 
   LICENSE ISSUES

be/src/common/kudu_version.cc

@@ -27,6 +27,11 @@ void VersionInfo::GetVersionInfoPB(VersionInfoPB* pb) {
   DCHECK(false);
 }
 
+std::string VersionInfo::GetShortVersionInfo() {
+  DCHECK(false);
+  return "";
+}
+
 std::string VersionInfo::GetAllVersionInfo() {
   DCHECK(false);
   return "";

be/src/kudu/util/CMakeLists.txt

@@ -160,7 +160,7 @@ set(UTIL_SRCS
   memory/overwrite.cc
   mem_tracker.cc
   metrics.cc
-  minidump.cc
+  # minidump.cc
   monotime.cc
   mutex.cc
   net/dns_resolver.cc
@@ -198,7 +198,9 @@ set(UTIL_SRCS
   trace_metrics.cc
   user.cc
   url-coding.cc
-  version_info.cc
+  # Remove from compilation, as it depends on generated method calls. Replaced by
+  # kudu_version.cc in Impala's common library.
+  # version_info.cc
   version_util.cc
   website_util.cc
   zlib.cc
@@ -282,7 +284,9 @@ add_library(kudu_test_util
 target_link_libraries(kudu_test_util
   gflags
   glog
-  gmock
+  # Impala doesn't have gmock in its toolchain
+  gtest
+  #gmock
   kudu_util)
 
 if(HAVE_LIB_VMEM)
@@ -329,7 +333,7 @@ endif()
 #######################################
 
 add_executable(protoc-gen-insertions protoc-gen-insertions.cc)
-target_link_libraries(protoc-gen-insertions gutil protobuf protoc ${KUDU_BASE_LIBS})
+target_link_libraries(protoc-gen-insertions gutil glog gflags protoc protobuf ${KUDU_BASE_LIBS})
 
 #######################################
 # Unit tests

be/src/kudu/util/compression/compression_codec.cc

@@ -23,6 +23,7 @@
 #include <vector>
 
 #include <glog/logging.h>
+#define LZ4_DISABLE_DEPRECATE_WARNINGS
 #include <lz4.h>
 #include <snappy-sinksource.h>
 #include <snappy.h>

be/src/kudu/util/flags.cc

@@ -74,9 +74,7 @@ DEFINE_bool(dump_metrics_json, false,
 TAG_FLAG(dump_metrics_json, hidden);
 
 #ifdef TCMALLOC_ENABLED
-DEFINE_bool(enable_process_lifetime_heap_profiling, false, "Enables heap "
-    "profiling for the lifetime of the process. Profile output will be stored in the "
-    "directory specified by -heap_profile_path.");
+DECLARE_bool(enable_process_lifetime_heap_profiling);
 TAG_FLAG(enable_process_lifetime_heap_profiling, stable);
 TAG_FLAG(enable_process_lifetime_heap_profiling, advanced);
 

be/src/kudu/util/kudu_export.h

@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// This file is generated during Kudu's build. Instead of recreating the necessary steps
+// in Impala's build process, we copy it into our repository. See
+// kudu/client/CMakeLists.txt in Kudu's repository for details.
+
+#ifndef KUDU_EXPORT_H
+#define KUDU_EXPORT_H
+
+#ifdef KUDU_STATIC_DEFINE
+#  define KUDU_EXPORT
+#  define KUDU_NO_EXPORT
+#else
+#  ifndef KUDU_EXPORT
+#    ifdef kudu_client_exported_EXPORTS
+        /* We are building this library */
+#      define KUDU_EXPORT __attribute__((visibility("default")))
+#    else
+        /* We are using this library */
+#      define KUDU_EXPORT __attribute__((visibility("default")))
+#    endif
+#  endif
+
+#  ifndef KUDU_NO_EXPORT
+#    define KUDU_NO_EXPORT __attribute__((visibility("hidden")))
+#  endif
+#endif
+
+#ifndef KUDU_DEPRECATED
+#  define KUDU_DEPRECATED __attribute__ ((__deprecated__))
+#endif
+
+#ifndef KUDU_DEPRECATED_EXPORT
+#  define KUDU_DEPRECATED_EXPORT KUDU_EXPORT KUDU_DEPRECATED
+#endif
+
+#ifndef KUDU_DEPRECATED_NO_EXPORT
+#  define KUDU_DEPRECATED_NO_EXPORT KUDU_NO_EXPORT KUDU_DEPRECATED
+#endif
+
+#if 0 /* DEFINE_NO_DEPRECATED */
+#  ifndef KUDU_NO_DEPRECATED
+#    define KUDU_NO_DEPRECATED
+#  endif
+#endif
+
+#endif

be/src/kudu/util/logging.cc

@@ -46,9 +46,7 @@
 #include "kudu/util/signal.h"
 #include "kudu/util/status.h"
 
-DEFINE_string(log_filename, "",
-    "Prefix of log filename - "
-    "full path is <log_dir>/<log_filename>.[INFO|WARN|ERROR|FATAL]");
+DECLARE_string(log_filename);
 TAG_FLAG(log_filename, stable);
 
 DEFINE_bool(log_async, true,
@@ -61,9 +59,7 @@ DEFINE_int32(log_async_buffer_bytes_per_level, 2 * 1024 * 1024,
              "level. Only relevant when --log_async is enabled.");
 TAG_FLAG(log_async_buffer_bytes_per_level, hidden);
 
-DEFINE_int32(max_log_files, 10,
-    "Maximum number of log files to retain per severity level. The most recent "
-    "log files are retained. If set to 0, all log files are retained.");
+DECLARE_int32(max_log_files);
 TAG_FLAG(max_log_files, runtime);
 TAG_FLAG(max_log_files, experimental);
 
@@ -272,7 +268,8 @@ void InitGoogleLoggingSafe(const char* arg) {
   IgnoreSigPipe();
 
   // For minidump support. Must be called before logging threads started.
-  CHECK_OK(BlockSigUSR1());
+  // Disabled by Impala, which does not link Kudu's minidump library.
+  // CHECK_OK(BlockSigUSR1());
 
   if (FLAGS_log_async) {
     EnableAsyncLogging();

be/src/kudu/util/minidump.cc

@@ -28,8 +28,8 @@
 #include <string>
 
 #if defined(__linux__)
-#include <breakpad/client/linux/handler/exception_handler.h>
-#include <breakpad/common/linux/linux_libc_support.h>
+#include <client/linux/handler/exception_handler.h>
+#include <common/linux/linux_libc_support.h>
 #endif // defined(__linux__)
 
 #include <gflags/gflags.h>
@@ -71,28 +71,18 @@ static bool ValidateMinidumpEnabled(const char* /*flagname*/, bool value) {
 }
 DEFINE_validator(enable_minidumps, &ValidateMinidumpEnabled);
 
-DEFINE_string(minidump_path, "minidumps", "Directory to write minidump files to. This "
-    "can be either an absolute path or a path relative to --log_dir. Each daemon will "
-    "create an additional sub-directory to prevent naming conflicts and to make it "
-    "easier to identify a crashing daemon. Minidump files contain crash-related "
-    "information in a compressed format. Minidumps will be written when a daemon exits "
-    "unexpectedly, for example on an unhandled exception or signal, or when a "
-    "SIGUSR1 signal is sent to the process. Cannot be set to an empty value.");
+DECLARE_string(minidump_path);
 TAG_FLAG(minidump_path, evolving);
 // The minidump path cannot be empty.
 static bool ValidateMinidumpPath(const char* /*flagname*/, const string& value) {
   return !value.empty();
 }
 DEFINE_validator(minidump_path, &ValidateMinidumpPath);
 
-DEFINE_int32(max_minidumps, 9, "Maximum number of minidump files to keep per daemon. "
-    "Older files are removed first. Set to 0 to keep all minidump files.");
+DECLARE_int32(max_minidumps);
 TAG_FLAG(max_minidumps, evolving);
 
-DEFINE_int32(minidump_size_limit_hint_kb, 20480, "Size limit hint for minidump files in "
-    "KB. If a minidump exceeds this value, then breakpad will reduce the stack memory it "
-    "collects for each thread from 8KB to 2KB. However it will always include the full "
-    "stack memory for the first 20 threads, including the thread that crashed.");
+DECLARE_int32(minidump_size_limit_hint_kb);
 TAG_FLAG(minidump_size_limit_hint_kb, advanced);
 TAG_FLAG(minidump_size_limit_hint_kb, evolving);
 

bin/rat_exclude_files.txt

@@ -40,6 +40,7 @@ www/d3.v3.min.js
 www/jquery/jquery-1.12.4.min.js
 tests/comparison/leopard/static/css/hljs.css
 tests/comparison/leopard/static/js/highlight.pack.js
+be/src/kudu/util/array_view.h
 be/src/kudu/util/cache-test.cc
 be/src/kudu/util/cache.cc
 be/src/kudu/util/cache.h