bazel/ci: unify developer-local and CI build generation.

WORKSPACE

@@ -1,80 +1,5 @@
 workspace(name = "envoy")
 
-load("//bazel:repositories.bzl", "envoy_dependencies")
+load("//bazel:prebuilt.bzl", "envoy_prebuilt_workspace_targets")
 
-envoy_dependencies()
-
-bind(
-    name = "ares",
-    actual = "@cares_git//:ares",
-)
-
-bind(
-    name = "cc_wkt_protos_genproto",
-    actual = "@protobuf_git//:cc_wkt_protos_genproto",
-)
-
-bind(
-    name = "cc_wkt_protos",
-    actual = "@protobuf_git//:cc_wkt_protos",
-)
-
-bind(
-    name = "event",
-    actual = "@libevent_git//:event",
-)
-
-bind(
-    name = "event_pthreads",
-    actual = "@libevent_git//:event_pthreads",
-)
-
-bind(
-    name = "googletest",
-    actual = "@googletest_git//:googletest",
-)
-
-bind(
-    name = "http_parser",
-    actual = "@http_parser_git//:http_parser",
-)
-
-bind(
-    name = "lightstep",
-    actual = "@lightstep_git//:lightstep_core",
-)
-
-bind(
-    name = "nghttp2",
-    actual = "@nghttp2_tar//:nghttp2",
-)
-
-bind(
-    name = "protobuf",
-    actual = "@protobuf_git//:protobuf",
-)
-
-bind(
-    name = "protoc",
-    actual = "@protobuf_git//:protoc",
-)
-
-bind(
-    name = "rapidjson",
-    actual = "@rapidjson_git//:rapidjson",
-)
-
-bind(
-    name = "spdlog",
-    actual = "@spdlog_git//:spdlog",
-)
-
-bind(
-    name = "ssl",
-    actual = "@boringssl//:ssl",
-)
-
-bind(
-    name = "tclap",
-    actual = "@tclap_archive//:tclap",
-)
+envoy_prebuilt_workspace_targets("//bazel")

bazel/BUILD

@@ -0,0 +1,5 @@
+package(default_visibility = ["//visibility:public"])
+
+load(":prebuilt.bzl", "envoy_prebuilt_targets")
+
+envoy_prebuilt_targets(prebuilts_genrule = True)

bazel/EXTERNAL_DEPS.md

@@ -0,0 +1,20 @@
+# Adding external dependencies to Envoy
+
+1. Specify name and version in [external documentation](../docs/install/requirements.rst).
+2. Add a build recipe in [`ci/build_container/build_recipes`](../ci/build_container/build_recipes)
+   for developer-local and CI external dependency build flows. Reference build recipe from the [`Makefile`](../ci/build_container/Makefile).
+3. Verify that the build recipe works by running `ENVOY_SRC_DIR=$PWD tools/setup_external_deps.sh`
+   from the Envoy root. The built artifacts are located in `build/prebuilt`.
+4. Add a `LIBS` entry in `bazel/gen_prebuilt.py` providing a reference to the build recipe and
+   a name X for the new external dependency target.
+5. `bazel/gen_prebuilt.sh` to rebuild `bazel/prebuilt.bzl`.
+6. Reference your new external dependency in some `envoy_cc_library` via X in the `external_deps`
+   attribute.
+7. `bazel test //test/...`
+
+# Updating an external dependency version
+
+1. Specify the new version in [external documentation](../docs/install/requirements.rst).
+2. Update the build recipe in [`ci/build_container/build_recipes`](../ci/build_container/build_recipes).
+3. `bazel/gen_prebuilt.sh` to rebuild `bazel/prebuilt.bzl`.
+4. `bazel test //test/...`

bazel/envoy_build_system.bzl

@@ -1,4 +1,4 @@
-load("@protobuf_git//:protobuf.bzl", "cc_proto_library")
+load("@protobuf_bzl//:protobuf.bzl", "cc_proto_library")
 
 ENVOY_COPTS = [
     # TODO(htuch): Remove this when Bazel bringup is done.

bazel/gen_prebuilt.py

@@ -0,0 +1,277 @@
+#!/usr/bin/env python
+
+# This tool produces a .bzl file that provides functions that export out as
+# cc_library and filegroup targets all of Envoy's external dependencies. At the
+# core of the generated .bzl file is a genrule that invokes the recursive make
+# system (inside a Bazel sandbox) located in ci/build_container/Makefile.
+#
+# A BUILD file with this genrule is generated, rather than prewritten, due to fundamental limits on
+# how Bazel builds its dependency graph. Bazel needs to know ahead of time, before it starts
+# executing rules, which outputs they produce. As a result, it needs to know, before invoking
+# ci/build_container/Makefile, what its resulting header/libraries will be - we can't use Bazel
+# globs as a result for example, since the results of ci/build_container/Makefile are not on
+# the filesystem at the time of glob evaluation.
+#
+# So, the approach taken here is that when a developer changes the external dependencies, they run
+# gen_prebuilt.sh, which first does a local build outside of Bazel with ci/build_container/Makefile.
+# Then, gen_prebuilt.py (this script) analyzes the files that resulted from the build and generates
+# a BUILD file, which then is checked into the repository.
+
+from collections import defaultdict
+from collections import namedtuple
+import os
+import string
+import sys
+
+# External libraries that we automatically discover headers from and generate
+# corresponding Bazel genrule and cc_library targets with.
+#   target: cc_library target name. This may be referenced in the external_deps
+#     parameter of envoy_cc_library.
+#   build_recipe: build recipe name in ci/build_container/build_recipes without
+#     .sh.
+#   build_paths: path in thirdparty_build/ of libraries to link.
+#   include_path: path to include files, starting with ${THIRDPARTY_BUILD} for
+#     generated headers or ${THIRDPARTY_SRC} for original source tree.
+Library = namedtuple('Library',
+                     ['target', 'build_recipe', 'build_paths', 'include_path'])
+LIBS = [
+    Library('ares', 'cares', ['lib/libcares.a'], '${THIRDPARTY_BUILD}/include'),
+    Library('crypto', 'boringssl', ['lib/libcrypto.a'],
+            '${THIRDPARTY_BUILD}/include'),
+    Library('event', 'libevent', ['lib/libevent.a'],
+            '${THIRDPARTY_BUILD}/include'),
+    Library('googletest', 'googletest', [
+        'lib/libgmock.a',
+        'lib/libgtest.a',
+    ], '${THIRDPARTY_BUILD}/include'),
+    Library('http_parser', 'http-parser', ['lib/libhttp_parser.a'],
+            '${THIRDPARTY_BUILD}/include'),
+    Library('lightstep_includes', 'lightstep', [],
+            '${THIRDPARTY_BUILD}/include'),
+    Library('nghttp2', 'nghttp2', ['lib/libnghttp2.a'],
+            '${THIRDPARTY_BUILD}/include'),
+    Library('protobuf', 'protobuf',
+            ['lib/libprotobuf.a', 'lib/libprotobuf-lite.a',
+             'lib/libprotoc.a'], '${THIRDPARTY_BUILD}/include'),
+    Library('rapidjson', 'rapidjson', [],
+            '${THIRDPARTY_SRC}/rapidjson-1.1.0/include'),
+    Library('spdlog', 'spdlog', [], '${THIRDPARTY_SRC}/spdlog-0.11.0/include'),
+    Library('tclap', 'tclap', [], '${THIRDPARTY_SRC}/tclap-1.2.1/include'),
+]
+
+# These library targets express dependency relationships in the Bazel graph so that the correct link
+# order can be expressed between .a files.
+DependentLibrary = namedtuple('DependentLibrary',
+                              ['target', 'build_recipe', 'build_paths', 'deps'])
+DEPLIBS = [
+    DependentLibrary('event_pthreads', 'libevent', ['lib/libevent_pthreads.a'],
+                     ['event']),
+    DependentLibrary('lightstep', 'lightstep',
+                     ['lib/liblightstep_core_cxx11.a'],
+                     ['lightstep_includes', 'protobuf']),
+    DependentLibrary('ssl', 'boringssl', ['lib/libssl.a'], ['crypto']),
+]
+
+# Filegroups are used to export misc. targets that are not libraries or headers, e.g.
+# the proto compiler.
+Filegroup = namedtuple('ExportedFile',
+                       ['target', 'build_recipe', 'build_paths'])
+FILEGROUPS = [
+    Filegroup('protoc', 'protobuf', ['bin/protoc']),
+]
+
+PREBUILT_TEMPLATE = string.Template(
+    '''# Everything in this file is generated by bazel/gen_prebuilt.sh, DO NOT HAND
+# EDIT. This script should be rerun on the change to any dependency.
+
+${list_def_defs}${generated_files}
+
+def envoy_prebuilt_targets(prebuilts_genrule=False):
+    if prebuilts_genrule:
+        native.genrule(
+            name = "prebuilts",
+            srcs = [
+                "//ci/build_container:build_and_install_deps.sh",
+                "//ci/build_container:prebuilt_build_inputs",
+            ],
+            outs = GENERATED_FILES,
+            cmd = "export GENDIR=$$$$(realpath $$(@D)); BUILD_DISTINCT=1 TMPDIR=$$$$GENDIR " +
+                  "THIRDPARTY_DEPS=$$$$GENDIR THIRDPARTY_SRC=$$$$GENDIR/thirdparty " +
+                  "THIRDPARTY_BUILD=$$$$GENDIR/thirdparty_build $$(location " +
+                  "//ci/build_container:build_and_install_deps.sh)",
+            message = "Building Envoy dependencies",
+        )
+
+${cc_libraries}
+${filegroups}
+def envoy_prebuilt_workspace_targets(path):
+${binds}
+    # Used only for protobuf.bzl
+    native.git_repository(
+        name = "protobuf_bzl",
+        # Using a non-canonical repository/branch here. This is a workaround to the lack of
+        # merge on https://github.com/google/protobuf/pull/2508, which is needed for supporting
+        # arbitrary CC compiler locations from the environment. The branch is
+        # https://github.com/htuch/protobuf/tree/v3.2.0-default-shell-env, which is the 3.2.0
+        # release with the above mentioned PR cherry picked.
+        commit = "d490587268931da78c942a6372ef57bb53db80da",
+        remote = "https://github.com/htuch/protobuf.git",
+    )
+''')
+
+SOURCE_DIR_NAME = 'thirdparty'
+BUILD_DIR_NAME = 'thirdparty_build'
+
+
+def GetSourceDir(build_recipe):
+  return os.path.join(SOURCE_DIR_NAME, '%s.dep' % build_recipe)
+
+
+def GetBuildDir(build_recipe):
+  return os.path.join(BUILD_DIR_NAME, '%s.dep' % build_recipe)
+
+
+# Obtain a dictionary with the thirdparty source/build paths. This will be relative to the runfiles
+# root unless a prefix is provided.
+def GetEnvPaths(lib, prefix_path=''):
+  return {
+      'THIRDPARTY_SRC':
+          os.path.join(prefix_path, GetSourceDir(lib.build_recipe)),
+      'THIRDPARTY_BUILD':
+          os.path.join(prefix_path, GetBuildDir(lib.build_recipe)),
+  }
+
+
+def SubstitueEnvPaths(lib, s):
+  return string.Template(s).substitute(GetEnvPaths(lib))
+
+
+# This is the heart of the generated output analysis. It walks the include directory of
+# the dependency and finds all the headers.
+def GetExportedIncludes(lib, prebuilt_path):
+  include_prefix = lib.include_path
+  env = GetEnvPaths(lib, prebuilt_path)
+  include_files = []
+  for root, dirs, files in os.walk(
+      string.Template(include_prefix).substitute(env)):
+    include_files.extend(
+        os.path.join(root[len(prebuilt_path) + 1:], f) for f in files)
+  return sorted(include_files)
+
+
+# The rest of this file is the pretty-printing of the BUILD file.
+
+
+def Indent(s, indent_level=0):
+  return '    ' * indent_level + s
+
+
+def Quote(x):
+  return '"%s"' % x
+
+
+def QuoteList(xs):
+  return map(Quote, xs)
+
+
+def FormatBazelValue(value, indent_level=0):
+  if not isinstance(value, list):
+    return str(value)
+  return '[\n' + '\n'.join('%s,' % Indent(str(v), indent_level + 1)
+                           for v in value) + '\n' + Indent(']', indent_level)
+
+
+def FormatBazelRule(name, parameters):
+  return Indent('%s(\n' % name, 1) + '\n'.join(
+      Indent('%s = %s,' % (key, FormatBazelValue(value, 2)), 2)
+      for key, value in parameters) + '\n' + Indent(')\n', 1)
+
+
+def HeadersName(name):
+  return '%s_HDRS' % name.upper()
+
+
+def LibsName(name):
+  return '%s_LIBS' % name.upper()
+
+
+def FormatListDefs(lib, include_files):
+  list_defs = []
+  if include_files:
+    list_defs.append('%s = %s' % (HeadersName(lib.target),
+                                  FormatBazelValue(QuoteList(include_files))))
+  if lib.build_paths:
+    lib_files = (os.path.join(GetBuildDir(lib.build_recipe), p)
+                 for p in lib.build_paths)
+    list_defs.append('%s = %s' % (LibsName(lib.target),
+                                  FormatBazelValue(QuoteList(lib_files))))
+  return '\n\n'.join(list_defs) + '\n\n'
+
+
+def FormatGeneratedFiles(libs, includes):
+  defs = []
+  for lib in libs:
+    if includes.get(lib.target, None):
+      defs.append(HeadersName(lib.target))
+    if hasattr(lib, 'build_paths') and lib.build_paths:
+      defs.append(LibsName(lib.target))
+  preamble = 'GENERATED_FILES = ('
+  separator = ' +\n' + ' ' * len(preamble)
+  return preamble + separator.join(defs) + ')'
+
+
+def FormatCcLibrary(lib, has_headers):
+  params = [('name', Quote(lib.target))]
+  if lib.build_paths:
+    params.append(('srcs', LibsName(lib.target)))
+  if has_headers:
+    params.append(('hdrs', HeadersName(lib.target)))
+  if hasattr(lib, 'deps'):
+    params.append(('deps', QuoteList(lib.deps)))
+  if hasattr(lib, 'include_path'):
+    params.append(('includes',
+                   [Quote(SubstitueEnvPaths(lib, lib.include_path))]))
+  return FormatBazelRule('native.cc_library', params)
+
+
+def FormatFilegroup(filegroup):
+  return FormatBazelRule('native.filegroup', [('name', Quote(filegroup.target)),
+                                              ('srcs',
+                                               LibsName(filegroup.target))])
+
+
+def FormatBind(lib):
+  return FormatBazelRule('native.bind',
+                         [('name', Quote(lib.target)),
+                          ('actual', 'path + ' + Quote(':%s' % lib.target))])
+
+
+if __name__ == '__main__':
+  if len(sys.argv) != 3:
+    print 'Usage: %s <prebuilt directory> <output BUILD path>' % sys.argv[0]
+    sys.exit(1)
+
+  prebuilt_path = sys.argv[1]
+  build_path = sys.argv[2]
+  includes = defaultdict(list)
+
+  for lib in sorted(LIBS):
+    includes[lib.target] = GetExportedIncludes(lib, prebuilt_path)
+
+  sorted_libs = sorted(LIBS + DEPLIBS, key=lambda lib: lib.target)
+  sorted_filegroups = sorted(FILEGROUPS, key=lambda lib: lib.target)
+  build_contents = PREBUILT_TEMPLATE.substitute(
+      list_def_defs=''.join(
+          FormatListDefs(lib, includes[lib.target])
+          for lib in sorted_libs + sorted_filegroups),
+      generated_files=FormatGeneratedFiles(sorted_libs + sorted_filegroups,
+                                           includes),
+      cc_libraries='\n'.join(
+          FormatCcLibrary(lib, bool(includes[lib.target]))
+          for lib in sorted_libs),
+      filegroups='\n'.join(FormatFilegroup(fg) for fg in sorted_filegroups),
+      binds='\n'.join(
+          FormatBind(lib) for lib in sorted_libs + sorted_filegroups))
+
+  with open(build_path, 'w') as f:
+    f.write(build_contents)