Workaround Travis CI build time problems. (#1801)

Until we have a better solution in place this enables us to bootstrap
the build cache. Basically when starting with a completely empty cache
the build will stop halfway, which will partially populate the cache and
make the next build go faster.

The majority of this change is to ensure that "halfway" captures enough
things to make for a substantial initial cache.

The algorithm to decide if the cache is "cold" is kind of naive, we
really test for "the cache has 0 bytes". Obviously this will not work
sometimes, for example, if the cache is full of garbage. In that case we
can manually clear the cache in Travis and start over.

I created #1800 to track a real solution, because these hacks are not
awesome.

CMakeLists.txt

@@ -139,6 +139,13 @@ set_property(CACHE GOOGLE_CLOUD_CPP_DEPENDENCY_PROVIDER
 set(PROJECT_THIRD_PARTY_DIR "${PROJECT_SOURCE_DIR}/third_party")
 list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
 
+# There are a number of targets that are pure dependencies, i.e., do not contain
+# our own code. These targets can be skipped when using static analysis tools.
+# Can be built earlier in the CI environment (for example to warm up the cache).
+# And in general can receive special treatment. This custom target collects all
+# these special targets with a single name.
+add_custom_target(google-cloud-cpp-dependencies)
+
 # Add gRPC and protobuf targets as submodules or packages, depending on the
 # configuration.  This should happen before enable_testing(), because we are not
 # interested in compiling and running the gRPC tests.
@@ -151,12 +158,5 @@ include(CTest)
 # Each subproject adds dependencies to this target to have their docs generated.
 add_custom_target(doxygen-docs)
 
-# There are a number of targets we cannot, or do not care to compile with the
-# static analyzer enabled. Either because the build times are too long, or
-# because the code is an external dependency, or generated code, or all of
-# above. These targets are added as dependencies of `skip-scanbuild-targets` and
-# compiled with, obviously, scan-build disabled.
-add_custom_target(skip-scanbuild-targets)
-
 # Add subprojects here.
 add_subdirectory(google/cloud)

ci/coverity-scan.sh

@@ -34,16 +34,9 @@ cmake -H. -B.coverity \
     -DCMAKE_BUILD_TYPE=Debug \
     -DGOOGLE_CLOUD_CPP_ENABLE_CCACHE=OFF
 
-# The project dependencies should be built without coverity-scan, any errors in
-# them are not actionable.
-cmake --build .coverity --target grpc_project -- -j $(nproc)
-cmake --build .coverity --target curl_project -- -j $(nproc)
-cmake --build .coverity --target crc32c_project -- -j $(nproc)
-cmake --build .coverity --target googletest_project -- -j $(nproc)
-
-# The proto-generated files contain too many errors, and they are not
-# actionable, so they are built without coverity-scan too.
-cmake --build .coverity --target skip-scanbuild-targets -- -j $(nproc)
+# The project dependencies, including the generated protos, should be built
+# without coverity-scan, any errors in them are not actionable.
+cmake --build .coverity --target google-cloud-cpp-dependencies -- -j $(nproc)
 
 # Run coverity scan over our code.
 cov-build --dir cov-int cmake --build .coverity -- -j $(nproc)

ci/travis/build-docker.sh

@@ -32,8 +32,7 @@ source "${PROJECT_ROOT}/ci/colors.sh"
 (cd "${PROJECT_ROOT}" ; ./ci/check-style.sh)
 
 # TODO (#1797): this is a workaround for the shared library error.
-readonly LD_LIBRARY_PATH="${PWD}/${BUILD_OUTPUT}/external/lib:${PWD}/${BUILD_OUTPUT}/external/lib64"
-export LD_LIBRARY_PATH
+export LD_LIBRARY_PATH="${PWD}/${BUILD_OUTPUT}/external/lib:${PWD}/${BUILD_OUTPUT}/external/lib64"
 
 CMAKE_COMMAND="cmake"
 if [ "${SCAN_BUILD}" = "yes" ]; then
@@ -42,6 +41,25 @@ fi
 
 ccache_command="$(which ccache)"
 
+if [[ -z "${ccache_command}" ]]; then
+  echo "The Travis builds cannot complete without ccache(1), exiting."
+  exit 1
+fi
+
+bootstrap_ccache="no"
+if ${ccache_command} --show-stats | grep '^cache size' | grep -q '0.0 kB'; then
+  echo "${COLOR_RED}"
+  echo "The ccache is empty. The builds cannot finish in the time allocated by"
+  echo "Travis without a warm cache. As a workaround, until #1800 is fixed,"
+  echo "the cache is bootstrapped in two steps. First, only a subset of the"
+  echo "system will be built, and the build will terminate with a failure."
+  echo "The second build on the same branch will have at least the previously"
+  echo "mentioned subset of the system in the build cache, and should be able"
+  echo "to finish in the time allocated by Travis."
+  echo "${COLOR_RESET}"
+  bootstrap_ccache="yes"
+fi
+
 echo "${COLOR_YELLOW}Started CMake config at: $(date)${COLOR_RESET}"
 echo "travis_fold:start:configure-cmake"
 # Tweak configuration for TEST_INSTALL=yes and SCAN_BUILD=yes builds.
@@ -84,11 +102,21 @@ fi
 echo "${COLOR_YELLOW}Started dependency build at: $(date)${COLOR_RESET}"
 echo "travis_fold:start:build-dependencies"
 echo
-cmake --build "${BUILD_OUTPUT}" --target skip-scanbuild-targets -- -j ${NCPU}
+cmake --build "${BUILD_OUTPUT}" \
+    --target google-cloud-cpp-dependencies -- -j ${NCPU}
 echo
 echo "travis_fold:end:build-dependencies"
 echo "${COLOR_YELLOW}Finished dependency build at: $(date)${COLOR_RESET}"
 
+if [[ "${bootstrap_ccache}" == "yes" ]]; then
+  echo
+  echo
+  echo "${COLOR_RED}Aborting the build early to warm up the cache.${COLOR_RESET}"
+  echo
+  echo
+  exit 1
+fi
+
 # If scan-build is enabled we build the smallest subset of things that is
 # needed; otherwise, we pick errors from things we do not care about. With
 # scan-build disabled we compile everything, to test the build as most

cmake/external/c-ares.cmake

@@ -58,6 +58,10 @@ if (NOT TARGET c_ares_project)
                         LOG_BUILD ON
                         LOG_INSTALL ON)
 
+    if (TARGET google-cloud-cpp-dependencies)
+        add_dependencies(google-cloud-cpp-dependencies c_ares_project)
+    endif ()
+
     add_library(c-ares::cares INTERFACE IMPORTED)
     set_library_properties_for_external_project(c-ares::cares cares
                                                 ALWAYS_SHARED)

cmake/external/crc32c.cmake

@@ -64,6 +64,10 @@ if (NOT TARGET crc32c_project)
                         LOG_BUILD ON
                         LOG_INSTALL ON)
 
+    if (TARGET google-cloud-cpp-dependencies)
+        add_dependencies(google-cloud-cpp-dependencies crc32c_project)
+    endif ()
+
     include(ExternalProjectHelper)
     add_library(Crc32c::crc32c INTERFACE IMPORTED)
     add_dependencies(Crc32c::crc32c crc32c_project)

cmake/external/curl.cmake

@@ -72,6 +72,10 @@ if (NOT TARGET curl_project)
         LOG_BUILD ON
         LOG_INSTALL ON)
 
+    if (TARGET google-cloud-cpp-dependencies)
+        add_dependencies(google-cloud-cpp-dependencies curl_project)
+    endif ()
+
     include(ExternalProjectHelper)
     add_library(CURL::CURL INTERFACE IMPORTED)
     add_dependencies(CURL::CURL curl_project)

cmake/external/googletest.cmake

@@ -65,6 +65,10 @@ if (NOT TARGET googletest_project)
                         LOG_BUILD ON
                         LOG_INSTALL ON)
 
+    if (TARGET google-cloud-cpp-dependencies)
+        add_dependencies(google-cloud-cpp-dependencies googletest_project)
+    endif ()
+
     # On Windows GTest uses library postfixes for debug versions, that is
     # gtest.lib becomes gtestd.lib when compiled with for debugging.  This ugly
     # expression computes that value. Note that it must be a generator

cmake/external/grpc.cmake

@@ -77,6 +77,10 @@ if (NOT TARGET gprc_project)
         LOG_BUILD ON
         LOG_INSTALL ON)
 
+    if (TARGET google-cloud-cpp-dependencies)
+        add_dependencies(google-cloud-cpp-dependencies grpc_project)
+    endif ()
+
     add_library(gRPC::address_sorting INTERFACE IMPORTED)
     set_library_properties_for_external_project(gRPC::address_sorting
                                                 address_sorting)

cmake/external/protobuf.cmake

@@ -76,6 +76,10 @@ if (NOT TARGET protobuf_project)
         LOG_BUILD ON
         LOG_INSTALL ON)
 
+    if (TARGET google-cloud-cpp-dependencies)
+        add_dependencies(google-cloud-cpp-dependencies protobuf_project)
+    endif ()
+
     add_library(protobuf::libprotobuf INTERFACE IMPORTED)
     add_dependencies(protobuf::libprotobuf protobuf_project)
     set_library_properties_for_external_project(protobuf::libprotobuf protobuf)

cmake/external/zlib.cmake

@@ -55,6 +55,10 @@ if (NOT TARGET zlib_project)
                         LOG_BUILD ON
                         LOG_INSTALL ON)
 
+    if (TARGET google-cloud-cpp-dependencies)
+        add_dependencies(google-cloud-cpp-dependencies zlib_project)
+    endif ()
+
     include(ExternalProjectHelper)
     add_library(ZLIB::ZLIB INTERFACE IMPORTED)
     add_dependencies(ZLIB::ZLIB zlib_project)

google/cloud/bigtable/CMakeLists.txt

@@ -119,7 +119,7 @@ add_library(bigtable::protos ALIAS bigtable_protos)
 # Enable unit tests
 include(CTest)
 
-add_dependencies(skip-scanbuild-targets bigtable_protos)
+add_dependencies(google-cloud-cpp-dependencies bigtable_protos)
 
 # Generate the version information from the CMake values.
 configure_file(version_info.h.in version_info.h)