Add all build targets to CMake and speed up builds

Use CMake's dependency resolution always instead of serial execution of
targets via shell scripts.  This improves parallelism by building fe,
be, and other targets at the same time and avoid some overhead from
invoking "make" multiple times. This reduces the time taken for
an incremental compilation of fe and be from 56s to 24s with this
command:

  ./buildall.sh -debug -noclean -notests -skiptests -ninja

Also use Impala-lzo's build script. This depends on the IMPALA-4277
fixes to the Impala-lzo build script.

Log directory creation is also moved from impala-config.sh to
buildall.sh. This means that impala-config.sh has no side-effects and
can be run concurrently with no issues.

Also make sure that "make" builds all the same artifacts as buildall.sh
when run with no args.

Testing:
Ran a jenkins core job, also experimented locally. Ran a jenkins core
job with distcc disabled - this exposed some concurrency bugs where
impala-config.sh fails if run concurrently.

Change-Id: I23617adf13bdeb034c24f6bba14b5ae480e8dd26
Reviewed-on: http://gerrit.cloudera.org:8080/4790
Reviewed-by: Tim Armstrong <tarmstrong@cloudera.com>
Tested-by: Impala Public Jenkins

CMakeLists.txt

@@ -305,21 +305,22 @@ add_subdirectory(be)
 add_subdirectory(fe)
 add_subdirectory(ext-data-source)
 
-# Run FE and BE tests
-add_custom_target(testall
-  COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_SOURCE_DIR}/fe mvn test
-  COMMAND ${CMAKE_SOURCE_DIR}/bin/runbackendtests.sh
-)
+add_custom_target(tarballs ALL DEPENDS shell_tarball)
 
-# Load test data
-add_custom_target(loadtestdata
-  COMMAND ${CMAKE_SOURCE_DIR}/bin/load-test-data.sh
+add_custom_target(shell_tarball DEPENDS thrift-deps
+  COMMAND "${CMAKE_SOURCE_DIR}/shell/make_shell_tarball.sh"
 )
 
-add_custom_target(benchmark_run
-  COMMAND ${CMAKE_SOURCE_DIR}/be/bin/run_hive_benchmark.py
+add_custom_target(cscope ALL
+  COMMAND "${CMAKE_SOURCE_DIR}/bin/gen-cscope.sh"
 )
 
+if (DEFINED ENV{IMPALA_LZO} AND EXISTS $ENV{IMPALA_LZO})
+  add_custom_target(impala-lzo ALL DEPENDS thrift-deps
+    COMMAND $ENV{IMPALA_LZO}/build.sh ${CMAKE_SOURCE_DIR} $ENV{IMPALA_TOOLCHAIN}
+  )
+endif()
+
 # Dump include paths to a file
 if (DUMP_INCLUDE_PATHS)
   file(REMOVE "${DUMP_INCLUDE_PATHS}")

be/src/benchmarks/CMakeLists.txt

@@ -22,9 +22,13 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/benchmarks")
 # where to put generated binaries
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/benchmarks")
 
+# Add custom target to only build the backend benchmarks
+add_custom_target(be-benchmarks)
+
 FUNCTION(ADD_BE_BENCHMARK BENCHMARK_NAME)
   ADD_EXECUTABLE(${BENCHMARK_NAME} ${BENCHMARK_NAME}.cc)
   TARGET_LINK_LIBRARIES(${BENCHMARK_NAME} ${IMPALA_LINK_LIBS})
+  ADD_DEPENDENCIES(be-benchmarks ${BENCHMARK_NAME})
 ENDFUNCTION()
 
 ADD_BE_BENCHMARK(atod-benchmark)

be/src/service/CMakeLists.txt

@@ -53,11 +53,11 @@ add_executable(impalad
 # All Impala daemons run from the same binary. The code that is run is determined by the
 # name (i.e. argv[0]) of the command that executes the binary, so we create symlinks for
 # statestored and catalogd.
-ADD_CUSTOM_TARGET(statestored
+ADD_CUSTOM_TARGET(statestored ALL
   ${CMAKE_COMMAND} -E create_symlink ${EXECUTABLE_OUTPUT_PATH}/impalad
   ${BUILD_OUTPUT_ROOT_DIRECTORY}/statestore/statestored DEPENDS impalad)
 
-ADD_CUSTOM_TARGET(catalogd
+ADD_CUSTOM_TARGET(catalogd ALL
   ${CMAKE_COMMAND} -E create_symlink ${EXECUTABLE_OUTPUT_PATH}/impalad
   ${BUILD_OUTPUT_ROOT_DIRECTORY}/catalog/catalogd DEPENDS impalad)
 

bin/impala-config.sh

@@ -228,25 +228,12 @@ export IMPALA_FE_TEST_COVERAGE_DIR="${IMPALA_FE_TEST_LOGS_DIR}/coverage"
 export IMPALA_BE_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/be_tests"
 export IMPALA_EE_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/ee_tests"
 export IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/custom_cluster_tests"
-# List of all Impala log dirs and create them.
+# List of all Impala log dirs so they can be created by buildall.sh
 export IMPALA_ALL_LOGS_DIRS="${IMPALA_CLUSTER_LOGS_DIR}
   ${IMPALA_DATA_LOADING_LOGS_DIR} ${IMPALA_DATA_LOADING_SQL_DIR}
   ${IMPALA_EE_TEST_LOGS_DIR} ${IMPALA_FE_TEST_COVERAGE_DIR}
   ${IMPALA_BE_TEST_LOGS_DIR} ${IMPALA_EE_TEST_LOGS_DIR}
   ${IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR}"
-mkdir -p $IMPALA_ALL_LOGS_DIRS
-
-# Create symlinks Testing/Temporary and be/Testing/Temporary that point to the BE test
-# log dir to capture the all logs of BE unit tests. Gtest has Testing/Temporary
-# hardwired in its code, so we cannot change the output dir by configuration.
-# We create two symlinks to capture the logs when running ctest either from
-# ${IMPALA_HOME} or ${IMPALA_HOME}/be.
-rm -rf "${IMPALA_HOME}/Testing"
-mkdir -p "${IMPALA_HOME}/Testing"
-ln -fs "${IMPALA_BE_TEST_LOGS_DIR}" "${IMPALA_HOME}/Testing/Temporary"
-rm -rf "${IMPALA_HOME}/be/Testing"
-mkdir -p "${IMPALA_HOME}/be/Testing"
-ln -fs "${IMPALA_BE_TEST_LOGS_DIR}" "${IMPALA_HOME}/be/Testing/Temporary"
 
 # Reduce the concurrency for local tests to half the number of cores in the system.
 # Note than nproc may not be available on older distributions (centos5.5)

bin/make_impala.sh

@@ -17,26 +17,33 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# Incrementally compiles the BE.
+# Incrementally compiles the frontend and backend.
 
 set -euo pipefail
 trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
 
 : ${IMPALA_TOOLCHAIN=}
 
+BUILD_EVERYTHING=1
+BUILD_FE_ONLY=0
 BUILD_TESTS=1
 CLEAN=0
 TARGET_BUILD_TYPE=${TARGET_BUILD_TYPE:-""}
 BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS:-""}
 CMAKE_ONLY=0
 MAKE_CMD=make
+MAKE_ARGS=-j${IMPALA_BUILD_THREADS:-4}
+
+# The minimal make targets if BUILD_EVERYTHING is 0.
+MAKE_TARGETS="impalad statestored catalogd fesupport loggingsupport ImpalaUdf"
 
 # parse command line options
 for ARG in $*
 do
   case "$ARG" in
     -notests)
       BUILD_TESTS=0
+      BUILD_EVERYTHING=0
       ;;
     -clean)
       CLEAN=1
@@ -56,16 +63,37 @@ do
     -cmake_only)
       CMAKE_ONLY=1
       ;;
+    -fe)
+      MAKE_TARGETS+=" fe"
+      ;;
+    -fe_only)
+      BUILD_FE_ONLY=1
+      BUILD_EVERYTHING=0
+      ;;
+    -cscope)
+      MAKE_TARGETS+=" cscope"
+      ;;
+    -impala-lzo)
+      MAKE_TARGETS+=" impala-lzo"
+      ;;
+    -tarballs)
+      MAKE_TARGETS+=" tarballs"
+      ;;
     -help|*)
       echo "make_impala.sh [-build_type=<build type> -notests -clean]"
       echo "[-build_type] : Target build type. Examples: Debug, Release, Address_sanitizer."
       echo "                If omitted, the last build target is built incrementally"
       echo "[-build_shared_libs] : Link all executables dynamically"
       echo "[-build_static_libs] : Link all executables statically (the default)"
       echo "[-cmake_only] : Generate makefiles and exit"
+      echo "[-fe] : Builds fe in addition to backend."
+      echo "[-fe_only] : Builds fe only."
       echo "[-ninja] : Use the Ninja build tool instead of Make"
-      echo "[-notests] : Omits building the tests."
+      echo "[-notests] : Omits building the backend tests and benchmarks."
       echo "[-clean] : Cleans previous build artifacts."
+      echo "[-cscope] : Builds cscope metadata."
+      echo "[-impala-lzo] : Builds Impala LZO."
+      echo "[-tarballs] : Builds additional tarballs like the shell tarball."
       echo ""
       echo "If either -build_type or -build_*_libs is set, cmake will be re-run for the "
       echo "project. Otherwise the last cmake configuration will continue to take effect."
@@ -74,6 +102,10 @@ do
   esac
 done
 
+if [ $BUILD_TESTS -eq 1 ]; then
+  MAKE_TARGETS+=" be-test be-benchmarks"
+fi
+
 echo "********************************************************************************"
 echo " Building Impala "
 if [ "x${TARGET_BUILD_TYPE}" != "x" ];
@@ -138,18 +170,11 @@ if [ $CMAKE_ONLY -eq 1 ]; then
   exit 0
 fi
 
-${MAKE_CMD} function-registry
-
-# With parallelism, make doesn't always make statestored and catalogd correctly if you
-# write make -jX impalad statestored catalogd. So we keep them separate and after impalad,
-# which they link to.
-${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4} impalad
-
-${MAKE_CMD} statestored
-${MAKE_CMD} catalogd
-if [ $BUILD_TESTS -eq 1 ]
-then
-  ${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4}
+MAKE_ARGS=-j${IMPALA_BUILD_THREADS:-4}
+if [ $BUILD_FE_ONLY -eq 1 ]; then
+  ${MAKE_CMD} ${MAKE_ARGS} fe
+elif [ $BUILD_EVERYTHING -eq 1 ]; then
+  ${MAKE_CMD} ${MAKE_ARGS}
 else
-  ${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4} fesupport loggingsupport ImpalaUdf
+  ${MAKE_CMD} ${MAKE_ARGS} ${MAKE_TARGETS}
 fi

buildall.sh

@@ -286,6 +286,23 @@ if [[ $TESTS_ACTION -eq 1 || $TESTDATA_ACTION -eq 1 || $FORMAT_CLUSTER -eq 1 ||
   NEED_MINICLUSTER=1
 fi
 
+create_log_dirs() {
+  # Create all of the log directories.
+  mkdir -p $IMPALA_ALL_LOGS_DIRS
+
+  # Create symlinks Testing/Temporary and be/Testing/Temporary that point to the BE test
+  # log dir to capture the all logs of BE unit tests. Gtest has Testing/Temporary
+  # hardwired in its code, so we cannot change the output dir by configuration.
+  # We create two symlinks to capture the logs when running ctest either from
+  # ${IMPALA_HOME} or ${IMPALA_HOME}/be.
+  rm -rf "${IMPALA_HOME}/Testing"
+  mkdir -p "${IMPALA_HOME}/Testing"
+  ln -fs "${IMPALA_BE_TEST_LOGS_DIR}" "${IMPALA_HOME}/Testing/Temporary"
+  rm -rf "${IMPALA_HOME}/be/Testing"
+  mkdir -p "${IMPALA_HOME}/be/Testing"
+  ln -fs "${IMPALA_BE_TEST_LOGS_DIR}" "${IMPALA_HOME}/be/Testing/Temporary"
+}
+
 bootstrap_dependencies() {
   # Populate necessary thirdparty components unless it's set to be skipped.
   if [[ "${SKIP_TOOLCHAIN_BOOTSTRAP}" = true ]]; then
@@ -310,37 +327,20 @@ bootstrap_dependencies() {
 
 # Build the Impala frontend and its dependencies.
 build_fe() {
-  "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS} -cmake_only
-  "${MAKE_CMD}" fe
+  "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS} -fe_only
 }
 
 # Build all components.
 build_all_components() {
-  # Build common and backend. This also sets up the CMake files.
-  echo "Calling make_impala.sh ${MAKE_IMPALA_ARGS}"
-  "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS}
-
+  # Build the Impala frontend, backend and external data source API.
+  MAKE_IMPALA_ARGS+=" -fe -cscope -tarballs"
   if [[ -e "$IMPALA_LZO" ]]
   then
-    pushd "$IMPALA_LZO"
-    LZO_CMAKE_ARGS+=" -DCMAKE_TOOLCHAIN_FILE=./cmake_modules/toolchain.cmake"
-    rm -f CMakeCache.txt
-    cmake ${LZO_CMAKE_ARGS}
-    "${MAKE_CMD}"
-    popd
+    MAKE_IMPALA_ARGS+=" -impala-lzo"
   fi
 
-  # Build the Java components (fe and external data source API).
-  pushd "$IMPALA_HOME"
-  "${MAKE_CMD}" ext-data-source fe
-  popd
-
-  # Build the shell tarball
-  echo "Creating shell tarball"
-  "${IMPALA_HOME}/shell/make_shell_tarball.sh"
-
-  # Generate list of files for Cscope to index
-  "$IMPALA_HOME/bin/gen-cscope.sh"
+  echo "Running make_impala.sh ${MAKE_IMPALA_ARGS}"
+  "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS}
 }
 
 # Do any configuration of the test cluster required by the script arguments.
@@ -426,6 +426,8 @@ if [[ "$CLEAN_ACTION" -eq 1 ]]; then
   "$IMPALA_HOME/bin/clean.sh"
 fi
 
+create_log_dirs
+
 bootstrap_dependencies
 
 if [[ "$BUILD_FE_ONLY" -eq 1 ]]; then

ext-data-source/CMakeLists.txt

@@ -15,6 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-add_custom_target(ext-data-source DEPENDS thrift-deps
+add_custom_target(ext-data-source ALL DEPENDS thrift-deps
   COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh install -DskipTests
 )

fe/CMakeLists.txt

@@ -15,6 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-add_custom_target(fe DEPENDS thrift-deps function-registry ext-data-source
+add_custom_target(fe ALL DEPENDS thrift-deps function-registry ext-data-source
   COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh install -DskipTests
 )