diff --git a/WORKSPACE b/WORKSPACE index add3951ebe0a..b993737d20ec 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -3,78 +3,3 @@ workspace(name = "envoy") load("//bazel:repositories.bzl", "envoy_dependencies") envoy_dependencies() - -bind( - name = "ares", - actual = "@cares_git//:ares", -) - -bind( - name = "cc_wkt_protos_genproto", - actual = "@protobuf_git//:cc_wkt_protos_genproto", -) - -bind( - name = "cc_wkt_protos", - actual = "@protobuf_git//:cc_wkt_protos", -) - -bind( - name = "event", - actual = "@libevent_git//:event", -) - -bind( - name = "event_pthreads", - actual = "@libevent_git//:event_pthreads", -) - -bind( - name = "googletest", - actual = "@googletest_git//:googletest", -) - -bind( - name = "http_parser", - actual = "@http_parser_git//:http_parser", -) - -bind( - name = "lightstep", - actual = "@lightstep_git//:lightstep_core", -) - -bind( - name = "nghttp2", - actual = "@nghttp2_tar//:nghttp2", -) - -bind( - name = "protobuf", - actual = "@protobuf_git//:protobuf", -) - -bind( - name = "protoc", - actual = "@protobuf_git//:protoc", -) - -bind( - name = "rapidjson", - actual = "@rapidjson_git//:rapidjson", -) - -bind( - name = "spdlog", - actual = "@spdlog_git//:spdlog", -) - -bind( - name = "ssl", - actual = "@boringssl//:ssl", -) - -bind( - name = "tclap", - actual = "@tclap_archive//:tclap", -) diff --git a/bazel/EXTERNAL_DEPS.md b/bazel/EXTERNAL_DEPS.md new file mode 100644 index 000000000000..baac0c4244c1 --- /dev/null +++ b/bazel/EXTERNAL_DEPS.md @@ -0,0 +1,18 @@ +# Adding external dependencies to Envoy + +1. Specify name and version in [external documentation](../docs/install/requirements.rst). +2. Add a build recipe X in [`ci/build_container/build_recipes`](../ci/build_container/build_recipes) + for developer-local and CI external dependency build flows. +3. Add the build recipe X to the list in [`bazel/recipes.bzl`](recipes.bzl). +4. Add a build target Y in [`ci/prebuilt/BUILD`](../ci/prebuilt/BUILD) to consume the headers and + libraries produced by the build recipe X. +5. Add a map from target Y to build recipe X in [`target_recipes.bzl`](target_recipes.bzl). +6. Reference your new external dependency in some `envoy_cc_library` via Y in the `external_deps` + attribute. +7. `bazel test //test/...` + +# Updating an external dependency version + +1. Specify the new version in [external documentation](../docs/install/requirements.rst). +2. Update the build recipe in [`ci/build_container/build_recipes`](../ci/build_container/build_recipes). +3. `bazel test //test/...` diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl index 9e0c35c519d3..3078656b763d 100644 --- a/bazel/envoy_build_system.bzl +++ b/bazel/envoy_build_system.bzl @@ -1,4 +1,4 @@ -load("@protobuf_git//:protobuf.bzl", "cc_proto_library") +load("@protobuf_bzl//:protobuf.bzl", "cc_proto_library") ENVOY_COPTS = [ # TODO(htuch): Remove this when Bazel bringup is done. diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 4219e2313e5b..2e3f4010d6c1 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -1,532 +1,90 @@ -# The build rules below for external dependencies build rules are maintained on a best effort basis. -# The rules are provided for developer convenience. For production builds, we recommend building the -# libraries according to their canonical build systems and expressing the dependencies in a manner -# similar to ci/WORKSPACE. - -def ares_repositories(): - BUILD = """ -cc_library( - name = "ares", - srcs = [ - "ares__close_sockets.c", - "ares__get_hostent.c", - "ares__read_line.c", - "ares__timeval.c", - "ares_cancel.c", - "ares_create_query.c", - "ares_data.c", - "ares_destroy.c", - "ares_expand_name.c", - "ares_expand_string.c", - "ares_fds.c", - "ares_free_hostent.c", - "ares_free_string.c", - "ares_getenv.c", - "ares_gethostbyaddr.c", - "ares_gethostbyname.c", - "ares_getnameinfo.c", - "ares_getopt.c", - "ares_getsock.c", - "ares_init.c", - "ares_library_init.c", - "ares_llist.c", - "ares_mkquery.c", - "ares_nowarn.c", - "ares_options.c", - "ares_parse_a_reply.c", - "ares_parse_aaaa_reply.c", - "ares_parse_mx_reply.c", - "ares_parse_naptr_reply.c", - "ares_parse_ns_reply.c", - "ares_parse_ptr_reply.c", - "ares_parse_soa_reply.c", - "ares_parse_srv_reply.c", - "ares_parse_txt_reply.c", - "ares_platform.c", - "ares_process.c", - "ares_query.c", - "ares_search.c", - "ares_send.c", - "ares_strcasecmp.c", - "ares_strdup.c", - "ares_strerror.c", - "ares_timeout.c", - "ares_version.c", - "ares_writev.c", - "bitncmp.c", - "inet_net_pton.c", - "inet_ntop.c", - "windows_port.c", - ], - hdrs = [ - "ares.h", - "ares_build.h", - "ares_config.h", - "ares_data.h", - "ares_dns.h", - "ares_getenv.h", - "ares_getopt.h", - "ares_inet_net_pton.h", - "ares_iphlpapi.h", - "ares_ipv6.h", - "ares_library_init.h", - "ares_llist.h", - "ares_nowarn.h", - "ares_platform.h", - "ares_private.h", - "ares_rules.h", - "ares_setup.h", - "ares_strcasecmp.h", - "ares_strdup.h", - "ares_version.h", - "ares_writev.h", - "bitncmp.h", - "nameser.h", - "setup_once.h", - ], - copts = [ - "-DHAVE_CONFIG_H", - ], - includes = ["."], - visibility = ["//visibility:public"], -) - -# There is some moderate evil here. buildconf needs to be invoked in its source -# location, where it also generates temp files (no option to place in TMPDIR). -# Also, configure does a cd relative dance that doesn't play nice with the -# symlink execroot in the Bazel build. So, copy everything prior to building. -ARES_CONFIG_SH_CMDS = [ - "export TMPDIR=$$(realpath $(@D))", - "rm -rf $(@D)/cares_build", - "mkdir -p $(@D)/cares_build", - "cp --parents -L $(SRCS) $(@D)/cares_build", - "pushd $(@D)/cares_build/external/cares_git", - "./buildconf", - "./configure", - "popd", - "cp $(@D)/cares_build/external/cares_git/ares_config.h $@", -] - -genrule( - name = "config", - srcs = glob(["**/*"]), - outs = ["ares_config.h"], - cmd = "&& ".join(ARES_CONFIG_SH_CMDS), -) - -genrule( - name = "ares_build", - srcs = ["ares_build.h.dist"], - outs = ["ares_build.h"], - cmd = "cp $(SRCS) $@", -) -""" - - native.new_git_repository( - name = "cares_git", - remote = "https://github.com/c-ares/c-ares.git", - commit = "7691f773af79bf75a62d1863fd0f13ebf9dc51b1", # v1.12.0 - build_file_content = BUILD, +load(":target_recipes.bzl", "TARGET_RECIPES") + +def _repository_impl(ctxt): + # Setup the build directory with links to the relevant files. + ctxt.symlink(Label("//bazel:repositories.sh"), "repositories.sh") + ctxt.symlink(Label("//ci/build_container:build_and_install_deps.sh"), + "build_and_install_deps.sh") + ctxt.symlink(Label("//ci/build_container:recipe_wrapper.sh"), "recipe_wrapper.sh") + ctxt.symlink(Label("//ci/build_container:Makefile"), "Makefile") + for r in ctxt.attr.recipes: + ctxt.symlink(Label("//ci/build_container/build_recipes:" + r + ".sh"), + "build_recipes/" + r + ".sh") + ctxt.symlink(Label("//ci/prebuilt:BUILD"), "BUILD") + + # Run the build script. + environment = {} + if ctxt.attr.debug: + environment["DEBUG"] = "1" + result = ctxt.execute( + ["./repositories.sh"] + ctxt.attr.recipes, + environment = environment, + # Ideally we would print progress, but instead this hangs on "INFO: Loading + # complete. Analyzing.." today, see + # https://github.com/bazelbuild/bazel/issues/1289. We could set quiet=False + # as well to indicate progress, but that isn't supported in versions folks + # are using right now (0.4.5). + # TODO(htuch): Revisit this when everyone is on newer Bazel versions. + # + # quiet = False, ) - -def boringssl_repositories(): - native.git_repository( - name = "boringssl", - commit = "bfd36df3da38dbf8828e712f42fbab2a0034bc40", # 2017-02-02 - remote = "https://boringssl.googlesource.com/boringssl", - ) - -def googletest_repositories(): - BUILD = """ -cc_library( - name = "googletest", - srcs = [ - "googlemock/src/gmock-all.cc", - "googletest/src/gtest-all.cc", - ], - hdrs = glob([ - "googlemock/include/**/*.h", - "googlemock/src/*.cc", - "googletest/include/**/*.h", - "googletest/src/*.cc", - "googletest/src/*.h", - ]), - includes = [ - "googlemock", - "googlemock/include", - "googletest", - "googletest/include", - ], - visibility = ["//visibility:public"], -) -""" - native.new_git_repository( - name = "googletest_git", - build_file_content = BUILD, - # v1.8.0 release - commit = "ec44c6c1675c25b9827aacd08c02433cccde7780", - remote = "https://github.com/google/googletest.git", - ) - -def http_parser_repositories(): - BUILD = """ -cc_library( - name = "http_parser", - srcs = [ - "http_parser.c", - ], - hdrs = [ - "http_parser.h", - ], - visibility = ["//visibility:public"], -) -""" - - native.new_git_repository( - name = "http_parser_git", - remote = "https://github.com/nodejs/http-parser.git", - commit = "9b0d5b33ebdaacff1dadd06bad4e198b11ff880e", - build_file_content = BUILD, - ) - -def libevent_repositories(): - BUILD = """ -genrule( - name = "config", - srcs = glob(["**/*"]), - outs = ["config.h"], - cmd = "TMPDIR=$(@D) $(location configure) --enable-shared=no --disable-libevent-regress " + - "--disable-openssl && cp config.h $@", - tools = ["configure"], -) - -genrule( - name = "event-config", - srcs = [ - "config.h", - "make-event-config.sed", - ], - outs = ["include/event2/event-config.h"], - cmd = "sed -f $(location make-event-config.sed) < $(location config.h) > $@", -) - -event_srcs = [ - "buffer.c", - "bufferevent.c", - "bufferevent_filter.c", - "bufferevent_pair.c", - "bufferevent_ratelim.c", - "bufferevent_sock.c", - "epoll.c", - "evdns.c", - "event.c", - "event_tagging.c", - "evmap.c", - "evrpc.c", - "evthread.c", - "evutil.c", - "evutil_rand.c", - "evutil_time.c", - "http.c", - "listener.c", - "log.c", - "poll.c", - "select.c", - "signal.c", - "strlcpy.c", - ":event-config", -] + glob(["*.h"]) - -event_pthread_srcs = [ - "evthread_pthread.c", - ":event-config", -] - -cc_library( - name = "event", - srcs = event_srcs, - hdrs = glob(["include/**/*.h"]) + [ - "arc4random.c", # arc4random.c is included by evutil_rand.c - "bufferevent-internal.h", - "defer-internal.h", - "evbuffer-internal.h", - "event-internal.h", - "evthread-internal.h", - "http-internal.h", - "iocp-internal.h", - "ipv6-internal.h", - "log-internal.h", - "minheap-internal.h", - "mm-internal.h", - "strlcpy-internal.h", - "util-internal.h", - "compat/sys/queue.h", - ], - copts = [ - "-w", - "-DHAVE_CONFIG_H", - ], - includes = [ - "compat", - "include", - ], - visibility = ["//visibility:public"], -) - -cc_library( - name = "event_pthreads", - srcs = event_pthread_srcs + ["include/event2/thread.h"], - hdrs = [ - "compat/sys/queue.h", - "evthread-internal.h", - ], - copts = [ - "-w", - "-DHAVE_CONFIG_H", - ], - includes = [ - "compat", - "include", - ], - visibility = ["//visibility:public"], - deps = [":event"], -)""" - - native.new_http_archive( - name = "libevent_git", - url = "https://github.com/libevent/libevent/releases/download/release-2.1.8-stable/libevent-2.1.8-stable.tar.gz", - strip_prefix = "libevent-2.1.8-stable", - build_file_content = BUILD, - ) - -def lightstep_repositories(): - BUILD = """ -load("@protobuf_git//:protobuf.bzl", "cc_proto_library") - -cc_library( - name = "lightstep_core", - srcs = [ - "src/c++11/impl.cc", - "src/c++11/span.cc", - "src/c++11/tracer.cc", - "src/c++11/util.cc", - ], - hdrs = [ - "src/c++11/lightstep/impl.h", - "src/c++11/lightstep/options.h", - "src/c++11/lightstep/propagation.h", - "src/c++11/lightstep/carrier.h", - "src/c++11/lightstep/span.h", - "src/c++11/lightstep/tracer.h", - "src/c++11/lightstep/util.h", - "src/c++11/lightstep/value.h", - "src/c++11/mapbox_variant/recursive_wrapper.hpp", - "src/c++11/mapbox_variant/variant.hpp", - ], - copts = [ - "-DPACKAGE_VERSION='\\"0.36\\"'", - "-Iexternal/lightstep_git/src/c++11/lightstep", - "-Iexternal/lightstep_git/src/c++11/mapbox_variant", - ], - includes = ["src/c++11"], - visibility = ["//visibility:public"], - deps = [ - "@lightstep_common_git//:collector_proto", - "@lightstep_common_git//:lightstep_carrier_proto", - "//external:protobuf", - ], -)""" - - COMMON_BUILD = """ -load("@protobuf_git//:protobuf.bzl", "cc_proto_library") - -cc_proto_library( - name = "collector_proto", - srcs = ["collector.proto"], - include = ".", - deps = [ - "//external:cc_wkt_protos", - ], - protoc = "//external:protoc", - default_runtime = "//external:protobuf", - visibility = ["//visibility:public"], -) - -cc_proto_library( - name = "lightstep_carrier_proto", - srcs = ["lightstep_carrier.proto"], - include = ".", - deps = [ - "//external:cc_wkt_protos", - ], - protoc = "//external:protoc", - default_runtime = "//external:protobuf", - visibility = ["//visibility:public"], -) -""" - - native.new_git_repository( - name = "lightstep_common_git", - remote = "https://github.com/lightstep/lightstep-tracer-common.git", - commit = "cbbecd671c1ae1f20ae873c5da688c8c14d04ec3", - build_file_content = COMMON_BUILD, - ) - - native.new_git_repository( - name = "lightstep_git", - remote = "https://github.com/lightstep/lightstep-tracer-cpp.git", - commit = "f1dc8f3dfd529350e053fd21273e627f409ae428", # 0.36 - build_file_content = BUILD, - ) - -def nghttp2_repositories(): - BUILD = """ -genrule( - name = "config", - srcs = glob(["**/*"]), - outs = ["config.h"], - cmd = "TMPDIR=$(@D) $(location configure) --enable-lib-only --enable-shared=no" + - " && cp config.h $@", - tools = ["configure"], -) - -cc_library( - name = "nghttp2", - srcs = glob([ - "lib/*.c", - "lib/*.h", - ]) + ["config.h"], - hdrs = glob(["lib/includes/nghttp2/*.h"]), - copts = [ - "-DHAVE_CONFIG_H", - "-DBUILDING_NGHTTP2", - ], - includes = [ - ".", - "lib/includes", - ], - visibility = ["//visibility:public"], -) -""" - - native.new_http_archive( - name = "nghttp2_tar", - url = "https://github.com/nghttp2/nghttp2/releases/download/v1.20.0/nghttp2-1.20.0.tar.gz", - strip_prefix = "nghttp2-1.20.0", - build_file_content = BUILD, - ) - -def protobuf_repositories(): - native.git_repository( - name = "protobuf_git", - # Using a non-canonical repository/branch here. This is a workaround to the lack of - # merge on https://github.com/google/protobuf/pull/2508, which is needed for supporting - # arbitrary CC compiler locations from the environment. The branch is - # https://github.com/htuch/protobuf/tree/v3.2.0-default-shell-env, which is the 3.2.0 - # release with the above mentioned PR cherry picked. - commit = "d490587268931da78c942a6372ef57bb53db80da", - remote = "https://github.com/htuch/protobuf.git", - ) - -def rapidjson_repositories(): - BUILD = """ -cc_library( - name = "rapidjson", - srcs = glob([ - "include/rapidjson/internal/*.h", - ]), - hdrs = glob([ - "include/rapidjson/*.h", - "include/rapidjson/error/*.h", - ]), - includes = ["include"], - visibility = ["//visibility:public"], -) -""" - - native.new_git_repository( - name = "rapidjson_git", - remote = "https://github.com/miloyip/rapidjson.git", - commit = "f54b0e47a08782a6131cc3d60f94d038fa6e0a51", # v1.1.0 - build_file_content = BUILD, - ) - -def spdlog_repositories(): - BUILD = """ -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "spdlog", - hdrs = glob([ - "include/**/*.cc", - "include/**/*.h", - ]), - strip_include_prefix = "include", -) -""" - - native.new_git_repository( - name = "spdlog_git", - build_file_content = BUILD, - # v0.11.0 release - commit = "1f1f6a5f3b424203a429e9cb78e6548037adefa8", - remote = "https://github.com/gabime/spdlog.git", + if result.return_code != 0 or ctxt.attr.debug: + print("External dep build exited with return code: %d" % result.return_code) + print(result.stdout) + print(result.stderr) + if result.return_code != 0: + fail("External dep build failed") + +def envoy_dependencies(path = "@envoy_deps//", skip_protobuf_bzl = False, skip_targets = []): + # Used only for protobuf.bzl. + if not skip_protobuf_bzl: + native.new_git_repository( + name = "protobuf_bzl", + # Using a non-canonical repository/branch here. This is a workaround to the lack of + # merge on https://github.com/google/protobuf/pull/2508, which is needed for supporting + # arbitrary CC compiler locations from the environment. The branch is + # https://github.com/htuch/protobuf/tree/v3.2.0-default-shell-env, which is the 3.2.0 + # release with the above mentioned PR cherry picked. + commit = "d490587268931da78c942a6372ef57bb53db80da", + remote = "https://github.com/htuch/protobuf.git", + # We only want protobuf.bzl, so don't support building out of this repo. + build_file_content = "", + ) + + # Set this to True to make the build debug cycles faster. + debug_build = False + + envoy_repository = repository_rule( + implementation = _repository_impl, + local = debug_build, + environ = [ + "CC", + "CXX", + "LD_LIBRARY_PATH" + ], + attrs = { + "debug": attr.bool(), + "recipes": attr.string_list(), + }, ) -def tclap_repositories(): - BUILD = """ -cc_library( - name = "tclap", - hdrs = [ - "include/tclap/Arg.h", - "include/tclap/ArgException.h", - "include/tclap/ArgTraits.h", - "include/tclap/CmdLine.h", - "include/tclap/CmdLineInterface.h", - "include/tclap/CmdLineOutput.h", - "include/tclap/Constraint.h", - "include/tclap/DocBookOutput.h", - "include/tclap/HelpVisitor.h", - "include/tclap/IgnoreRestVisitor.h", - "include/tclap/MultiArg.h", - "include/tclap/MultiSwitchArg.h", - "include/tclap/OptionalUnlabeledTracker.h", - "include/tclap/StandardTraits.h", - "include/tclap/StdOutput.h", - "include/tclap/SwitchArg.h", - "include/tclap/UnlabeledMultiArg.h", - "include/tclap/UnlabeledValueArg.h", - "include/tclap/ValueArg.h", - "include/tclap/ValuesConstraint.h", - "include/tclap/VersionVisitor.h", - "include/tclap/Visitor.h", - "include/tclap/XorHandler.h", - "include/tclap/ZshCompletionOutput.h", - ], - defines = [ - "HAVE_LONG_LONG=1", - "HAVE_SSTREAM=1", - ], - includes = ["include"], - visibility = ["//visibility:public"], -) -""" - native.new_http_archive( - name = "tclap_archive", - url = "https://storage.googleapis.com/istio-build-deps/tclap-1.2.1.tar.gz", - strip_prefix = "tclap-1.2.1", - build_file_content = BUILD, + # Ideally, we wouldn't have a single repository target for all dependencies, but instead one per + # dependency, as suggested in #747. However, it's much faster to build all deps under a single + # recursive make job and single make jobserver. + recipes = depset() + for t in TARGET_RECIPES: + if t not in skip_targets: + recipes += depset([TARGET_RECIPES[t]]) + + envoy_repository( + name = "envoy_deps", + debug = debug_build, + recipes = recipes.to_list(), ) -def envoy_dependencies(): - ares_repositories() - boringssl_repositories() - googletest_repositories() - http_parser_repositories() - libevent_repositories() - lightstep_repositories() - nghttp2_repositories() - protobuf_repositories() - rapidjson_repositories() - spdlog_repositories() - tclap_repositories() + for t in TARGET_RECIPES: + if t not in skip_targets: + native.bind( + name = t, + actual = path + ":" + t, + ) diff --git a/bazel/repositories.sh b/bazel/repositories.sh new file mode 100755 index 000000000000..499bd80b7546 --- /dev/null +++ b/bazel/repositories.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -e + +# When debugging the build, it's helpful to keep the artifacts in /tmp, since they don't get +# repeatedly clobbered as build recipes are modified. This is controlled by debug_build in +# bazel/respositories.bzl. +if [[ "${DEBUG}" == "1" ]] +then + BASEDIR=/tmp/bazel-envoy-deps + # Tell build_and_install_deps.sh to build sequentially when performance debugging. + # export BUILD_CONCURRENCY=0 +else + BASEDIR="${PWD}/build" +fi + +mkdir -p "${BASEDIR}" + +export THIRDPARTY_DEPS="${BASEDIR}" +export THIRDPARTY_SRC="${BASEDIR}/thirdparty" +export THIRDPARTY_BUILD="${BASEDIR}/thirdparty_build" + +DEPS="" +for r in "$@" +do + DEPS="${DEPS} ${THIRDPARTY_DEPS}/$r.dep" +done + +set -o pipefail +(time ./build_and_install_deps.sh ${DEPS}) 2>&1 | \ + tee "${BASEDIR}"/build.log + +# Need to rsync in debug mode, since the symlinks are into /tmp and cause problems with later build +# sandboxing. +if [[ "${DEBUG}" == "1" ]] +then + rsync -a "$(realpath "${THIRDPARTY_SRC}")"/ thirdparty + rsync -a "$(realpath "${THIRDPARTY_BUILD}")"/ thirdparty_build +else + ln -sf "$(realpath "${THIRDPARTY_SRC}")" thirdparty + ln -sf "$(realpath "${THIRDPARTY_BUILD}")" thirdparty_build +fi diff --git a/bazel/target_recipes.bzl b/bazel/target_recipes.bzl new file mode 100644 index 000000000000..42ad772c2283 --- /dev/null +++ b/bazel/target_recipes.bzl @@ -0,0 +1,24 @@ +# These should reflect //ci/prebuilt/BUILD declared targets. This a map from +# target in //ci/prebuilt/BUILD to the underlying build recipe in +# ci/build_container/build_recipes. +TARGET_RECIPES = { + "ares": "cares", + # TODO(htuch): Remove when cmake goes. + "cotire": "cotire", + "event": "libevent", + "event_pthreads": "libevent", + # TODO(htuch): This shouldn't be a build recipe, it's a tooling dependency + # that is external to Bazel. + "gcovr": "gcovr", + "googletest": "googletest", + "gperftools": "gperftools", + "http_parser": "http-parser", + "lightstep": "lightstep", + "nghttp2": "nghttp2", + "protobuf": "protobuf", + "protoc": "protobuf", + "rapidjson": "rapidjson", + "spdlog": "spdlog", + "ssl": "boringssl", + "tclap": "tclap", +} diff --git a/ci/WORKSPACE b/ci/WORKSPACE index 4cfc2a6c7b62..1fb35678df75 100644 --- a/ci/WORKSPACE +++ b/ci/WORKSPACE @@ -1,71 +1,15 @@ workspace(name = "ci") -bind( - name = "ares", - actual = "//ci/prebuilt:ares", -) - -bind( - name = "event", - actual = "//ci/prebuilt:event", -) - -bind( - name = "event_pthreads", - actual = "//ci/prebuilt:event_pthreads", -) - -bind( - name = "googletest", - actual = "//ci/prebuilt:googletest", -) - -bind( - name = "http_parser", - actual = "//ci/prebuilt:http_parser", -) +load("//bazel:repositories.bzl", "envoy_dependencies") -bind( - name = "lightstep", - actual = "//ci/prebuilt:lightstep", +envoy_dependencies( + path = "//ci/prebuilt", + skip_protobuf_bzl = True, ) -bind( - name = "nghttp2", - actual = "//ci/prebuilt:nghttp2", -) - -bind( - name = "protobuf", - actual = "//ci/prebuilt:protobuf", -) - -local_repository( - name = "protobuf_git", +new_local_repository( + name = "protobuf_bzl", path = "/thirdparty/protobuf-3.2.0", -) - -bind( - name = "protoc", - actual = "//ci/prebuilt:protoc", -) - -bind( - name = "rapidjson", - actual = "//ci/prebuilt:rapidjson", -) - -bind( - name = "spdlog", - actual = "//ci/prebuilt:spdlog", -) - -bind( - name = "ssl", - actual = "//ci/prebuilt:ssl", -) - -bind( - name = "tclap", - actual = "//ci/prebuilt:tclap", + # We only want protobuf.bzl, so don't support building out of this repo. + build_file_content = "", ) diff --git a/ci/WORKSPACE.consumer b/ci/WORKSPACE.consumer index 69f2f1820822..e7102ae72c8c 100644 --- a/ci/WORKSPACE.consumer +++ b/ci/WORKSPACE.consumer @@ -5,72 +5,6 @@ local_repository( path = "/source", ) -bind( - name = "ares", - actual = "@envoy//ci/prebuilt:ares", -) - -bind( - name = "event", - actual = "@envoy//ci/prebuilt:event", -) - -bind( - name = "event_pthreads", - actual = "@envoy//ci/prebuilt:event_pthreads", -) - -bind( - name = "googletest", - actual = "@envoy//ci/prebuilt:googletest", -) +load("//bazel:repositories.bzl", "envoy_dependencies") -bind( - name = "http_parser", - actual = "@envoy//ci/prebuilt:http_parser", -) - -bind( - name = "lightstep", - actual = "@envoy//ci/prebuilt:lightstep", -) - -bind( - name = "nghttp2", - actual = "@envoy//ci/prebuilt:nghttp2", -) - -bind( - name = "protobuf", - actual = "@envoy//ci/prebuilt:protobuf", -) - -local_repository( - name = "protobuf_git", - path = "/thirdparty/protobuf-3.2.0", -) - -bind( - name = "protoc", - actual = "@envoy//ci/prebuilt:protoc", -) - -bind( - name = "rapidjson", - actual = "@envoy//ci/prebuilt:rapidjson", -) - -bind( - name = "spdlog", - actual = "@envoy//ci/prebuilt:spdlog", -) - -bind( - name = "ssl", - actual = "@envoy//ci/prebuilt:ssl", -) - -bind( - name = "tclap", - actual = "@envoy//ci/prebuilt:tclap", -) +envoy_dependencies(path = "@envoy//ci/prebuilt") diff --git a/ci/build_container/BUILD b/ci/build_container/BUILD new file mode 100644 index 000000000000..4f42f1a03c99 --- /dev/null +++ b/ci/build_container/BUILD @@ -0,0 +1,6 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "build_and_install_deps.sh", + "Makefile", +]) diff --git a/ci/build_container/Dockerfile b/ci/build_container/Dockerfile index aaafd91ff067..4791b8e36515 100644 --- a/ci/build_container/Dockerfile +++ b/ci/build_container/Dockerfile @@ -1,4 +1,4 @@ FROM ubuntu:xenial -COPY ./build_container.sh ./build_and_install_deps.sh ./Makefile / +COPY ./build_container.sh ./build_and_install_deps.sh ./target_recipes.bzl ./recipe_wrapper.sh ./Makefile / COPY ./build_recipes/*.sh /build_recipes/ RUN ./build_container.sh diff --git a/ci/build_container/Makefile b/ci/build_container/Makefile index c3c570d54b91..14ad66addfb0 100644 --- a/ci/build_container/Makefile +++ b/ci/build_container/Makefile @@ -2,23 +2,12 @@ # version number, etc.) to uniquely identify the revision of the upstream dependency. This allows # make to pick up changes with a simple direct dependency on the build recipe. -all: $(THIRDPARTY_DEPS)/libevent.dep \ - $(THIRDPARTY_DEPS)/boringssl.dep \ - $(THIRDPARTY_DEPS)/gperftools.dep \ - $(THIRDPARTY_DEPS)/nghttp2.dep \ - $(THIRDPARTY_DEPS)/cares.dep \ - $(THIRDPARTY_DEPS)/protobuf.dep \ - $(THIRDPARTY_DEPS)/cotire.dep \ - $(THIRDPARTY_DEPS)/spdlog.dep \ - $(THIRDPARTY_DEPS)/http-parser.dep \ - $(THIRDPARTY_DEPS)/tclap.dep \ - $(THIRDPARTY_DEPS)/lightstep.dep \ - $(THIRDPARTY_DEPS)/rapidjson.dep \ - $(THIRDPARTY_DEPS)/googletest.dep \ - $(THIRDPARTY_DEPS)/gcovr.dep - RECIPES := build_recipes +# Make sure we use a consistent compiler across all deps. +CC ?= gcc +CXX ?= g++ + # If $(BUILD_DISTINCT) is set in the make environment, the artifacts are built and installed in # distinct directories under $(THIRDPARTY_BUILD) and $(THIRDPARTY_SRC). They end up looking like # $(THIRDPARTY_BUILD)/protobuf.dep/include, $(THIRDPARTY_BUILD)/cotire.dep/include etc. instead of @@ -44,8 +33,10 @@ build-recipe = cd $(THIRDPARTY_SRC) && \ $(build-setup) && \ (((THIRDPARTY_SRC=$(THIRDPARTY_SRC)/$(DISTINCT_PATH) \ THIRDPARTY_BUILD=$(THIRDPARTY_BUILD)/$(DISTINCT_PATH) \ + CC=$(CC) \ + CXX=$(CXX) \ $(1) \ - bash $(realpath $<) 2>&1) > $@.log) || (cat $@.log; exit 1)) && \ + time bash $(CURDIR)/recipe_wrapper.sh $(realpath $<) 2>&1) > $@.log) || (cat $@.log; exit 1)) && \ $(build-complete) $(THIRDPARTY_DEPS)/%.dep: $(RECIPES)/%.sh diff --git a/ci/build_container/build_and_install_deps.sh b/ci/build_container/build_and_install_deps.sh index ad026f19ed0f..26c12a01c1b2 100755 --- a/ci/build_container/build_and_install_deps.sh +++ b/ci/build_container/build_and_install_deps.sh @@ -7,4 +7,15 @@ mkdir -p "${THIRDPARTY_BUILD}" mkdir -p "${THIRDPARTY_SRC}" NUM_CPUS=`grep -c ^processor /proc/cpuinfo` -make -C "$(dirname "$0")" -j "${NUM_CPUS}" + +# Invokers can set BUILD_CONCURRENCY=0 to ensure each build recipe is invoked sequentially, with all +# CPU resources available. This is useful when debugging build performance. +if [[ "${BUILD_CONCURRENCY}" == "0" ]] +then + for dep in "$@" + do + make -C "$(dirname "$0")" -j "${NUM_CPUS}" "$dep" + done +else + make -C "$(dirname "$0")" -j "${NUM_CPUS}" "$@" +fi diff --git a/ci/build_container/build_container.sh b/ci/build_container/build_container.sh index 4a32dea6068d..314319e13bb3 100755 --- a/ci/build_container/build_container.sh +++ b/ci/build_container/build_container.sh @@ -4,7 +4,8 @@ set -e # Setup basic requirements and install them. apt-get update -apt-get install -y wget software-properties-common make cmake git python python-pip clang-format-3.6 bc libtool automake lcov zip +apt-get install -y wget software-properties-common make cmake git python python-pip \ + clang-format-3.6 bc libtool automake zip time apt-get install -y golang # For debugging. apt-get install -y gdb strace @@ -44,10 +45,7 @@ export CXX=g++-4.9 export THIRDPARTY_DEPS=/tmp export THIRDPARTY_SRC=/thirdparty export THIRDPARTY_BUILD=/thirdparty_build -# TODO(htuch): Remove the first build of the libraries in non-distinct locations when cmake is gone. -# Below we now build/install twice and this requires 2x the space in the Docker image as is to -# support both Bazel and cmake, but it's not worth fixing up all the cmake stuff since it's going -# soon. -"$(dirname "$0")"/build_and_install_deps.sh -rm -f /tmp/*.dep -BUILD_DISTINCT=1 "$(dirname "$0")"/build_and_install_deps.sh +DEPS=$(python <(cat target_recipes.bzl; \ + echo "print ' '.join(\"${THIRDPARTY_DEPS}/%s.dep\" % r for r in set(TARGET_RECIPES.values()))")) +echo "Building deps ${DEPS}" +"$(dirname "$0")"/build_and_install_deps.sh ${DEPS} diff --git a/ci/build_container/build_recipes/BUILD b/ci/build_container/build_recipes/BUILD new file mode 100644 index 000000000000..b1f93d428065 --- /dev/null +++ b/ci/build_container/build_recipes/BUILD @@ -0,0 +1,3 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files(glob(["*.sh"])) diff --git a/ci/build_container/docker_build.sh b/ci/build_container/docker_build.sh new file mode 100755 index 000000000000..e93e2aa50af6 --- /dev/null +++ b/ci/build_container/docker_build.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# We need target_recipes.bzl for the build, but it's not in ci/build_container. Using Docker +# relative path workaround from https://github.com/docker/docker/issues/2745#issuecomment-253230025 +# to get this to work. +tar cf - . -C ../../bazel target_recipes.bzl | docker build --rm -t lyft/envoy-build:$TAG - diff --git a/ci/build_container/recipe_wrapper.sh b/ci/build_container/recipe_wrapper.sh new file mode 100644 index 000000000000..f35844d7338d --- /dev/null +++ b/ci/build_container/recipe_wrapper.sh @@ -0,0 +1,6 @@ +PS4='+ $(date "+%s.%N") ' +set -x + +. $1 + +echo DONE diff --git a/ci/build_container/update_build_container.sh b/ci/build_container/update_build_container.sh index 2dcad425d6a0..bfdd519fc51d 100755 --- a/ci/build_container/update_build_container.sh +++ b/ci/build_container/update_build_container.sh @@ -1,11 +1,12 @@ #!/bin/bash + read -r -p "Do you have master checked out with most recent changes? [y/N] " response if [[ $response =~ ^([yY][eE][sS]|[yY])$ ]] then TAG="$(git rev-parse master)" docker-machine start default eval $(docker-machine env default) - docker build --rm -t lyft/envoy-build:$TAG . + ./docker_build.sh docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD docker push lyft/envoy-build:$TAG echo Pushed lyft/envoy-build:$TAG diff --git a/tools/build_profile.py b/tools/build_profile.py new file mode 100755 index 000000000000..debf0411d70b --- /dev/null +++ b/tools/build_profile.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +# This tool take the foo.dep.log output from a build recipe run under recipe_wrapper.sh on stdin and +# produces a profile of command execution time on the stdout. + +import re +import sys + +def PrintProfile(f): + prev_cmd = None + prev_timestamp = None + for line in f: + sr = re.match('\++ (\d+\.\d+) (.*)', line) + if sr: + timestamp, cmd = sr.groups() + if prev_cmd: + print '%.2f %s' % (float(timestamp) - float(prev_timestamp), prev_cmd) + prev_timestamp, prev_cmd = timestamp, cmd + +if __name__ == '__main__': + PrintProfile(sys.stdin)