diff --git a/.gitignore b/.gitignore index eef29c1dc..2f836aacf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ *~ *.pyc - diff --git a/.travis.yml b/.travis.yml index 884aa8f5d..a2c31d881 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,4 +21,4 @@ script: - sudo apt-get install python2.7 # We don't want to write the script inline because of Travis kludginess -- # it looks like it escapes " and \ in scripts when using `matrix:`. - - ./build/travis_docker_build.sh + - ./build/fbcode_builder/travis_docker_build.sh diff --git a/README.md b/README.md index 5c92b9061..a9fc77667 100644 --- a/README.md +++ b/README.md @@ -41,16 +41,16 @@ The CLI tools and web UI will be shipping shortly. Bistro needs a 64-bit Linux, Folly, FBThrift, Proxygen, boost, and libsqlite3. You need 2-3GB of RAM to build, as well as GCC 4.9 or above. -`bistro/build/ci/README` documents the usage of Docker-based scripts that -build Bistro on Ubuntu 14.04, 16.04, and Debian 8.6. You should be able to -follow very similar steps on most modern Linux distributions. +`build/README.md` documents the usage of Docker-based scripts that build +Bistro on Ubuntu 14.04, 16.04, and Debian 8.6. You should be able to follow +very similar steps on most modern Linux distributions. -If you run into dependency problems, look at bistro/build/CMakeLists.txt for +If you run into dependency problems, look at `bistro/cmake/setup.cmake` for a full list of Bistro's external dependencies (direct and indirect). We gratefully accept patches that improve Bistro's builds, or add support for various flavors of Linux and Mac OS. -The binaries will be in bistro/build/{Debug,Release}. Available build +The binaries will be in bistro/cmake/{Debug,Release}. Available build targets are explained here: http://cmake.org/Wiki/CMake_Useful_Variables#Compilers_and_Tools You can start Bistro's unit tests by running `ctest` in those directories. @@ -88,13 +88,13 @@ Open two terminals, one for the scheduler, and one for the worker. # In both terminals cd bistro/bistro # Start the scheduler in one terminal -./build/Debug/server/bistro_scheduler \ +./cmake/Debug/server/bistro_scheduler \ --server_port=6789 --http_server_port=6790 \ --config_file=scripts/test_configs/simple --clean_statuses \ --CAUTION_startup_wait_for_workers=1 --instance_node_name=scheduler # Start the worker in another mkdir /tmp/bistro_worker -./build/Debug/worker/bistro_worker --server_port=27182 --scheduler_host=:: \ +./cmake/Debug/worker/bistro_worker --server_port=27182 --scheduler_host=:: \ --scheduler_port=6789 --worker_command="$HOME/demo_bistro_task.sh" \ --data_dir=/tmp/bistro_worker ``` @@ -113,7 +113,7 @@ put the SQLite database, via `--data_dir=/tmp/bistro_scheduler` and ``` mkdir /tmp/bistro_scheduler -./build/Debug/server/bistro_scheduler \ +./cmake/Debug/server/bistro_scheduler \ --server_port=6789 --http_server_port=6790 \ --config_file=scripts/test_configs/simple \ --data_dir=/tmp/bistro_scheduler --status_table=task_statuses \ @@ -140,7 +140,7 @@ less scripts/test_configs/simple For debugging, we typically invoke the binaries like this: ``` -gdb build/Debug/worker/bistro_worker -ex "r ..." 2>&1 | tee WORKER.txt +gdb cmake/Debug/worker/bistro_worker -ex "r ..." 2>&1 | tee WORKER.txt ``` When configuring a real deployment, be sure to carefully review the `--help` diff --git a/bistro/build/ci/build_steps.py b/bistro/build/ci/build_steps.py deleted file mode 100644 index 61af4dcdb..000000000 --- a/bistro/build/ci/build_steps.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -''' - -An fbcode_builder program to install Bistro's dependencies, and then to -build & test Bistro. - -''' - -from fbcode_builder.fbcode_builder import ShellQuoted - - -def build_bistro(builder): - # These APIs should not change, so build a tag instead of building master. - for p in ('facebook/zstd', 'no1msd/mstch'): - builder.add_option( - p + ':git_hash', - ShellQuoted('$(git describe --abbrev=0 --tags)') - ) - - # Speeds up the build - builder.add_option('wangle/wangle:cmake_defines', {'BUILD_TESTS': 'OFF'}) - - return builder.build([ - builder.fb_github_autoconf_install('folly/folly'), - builder.fb_github_cmake_install('wangle/wangle'), - builder.fb_github_autoconf_install('proxygen/proxygen'), - # Thrift and dependencies - builder.github_project_workdir('facebook/zstd', '.'), - builder.zstd_make_and_install(), - builder.github_project_workdir('no1msd/mstch', '.'), - builder.cmake_install('no1msd/mstch'), - builder.fb_github_autoconf_install('fbthrift/thrift'), - # Bistro - builder.fb_github_project_workdir('bistro/bistro'), - builder.step('Build bistro', [ - # Future: should this share some code with `cmake_install()`? - builder.run(ShellQuoted( - 'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages ' - './build/run-cmake.sh Debug -DCMAKE_INSTALL_PREFIX={p}' - ).format(p=builder.option('prefix'))), - builder.workdir('build/Debug'), - builder.parallel_make(), - ]), - builder.step('Run bistro tests', [ - builder.run(ShellQuoted('ctest')), - ]) - ]) diff --git a/bistro/build/ci/fbcode_builder b/bistro/build/ci/fbcode_builder deleted file mode 120000 index 3f9b9b22c..000000000 --- a/bistro/build/ci/fbcode_builder +++ /dev/null @@ -1 +0,0 @@ -../../../../opensource/fbcode_builder \ No newline at end of file diff --git a/bistro/build/ci/make_docker_context.py b/bistro/build/ci/make_docker_context.py deleted file mode 100755 index dcba6476b..000000000 --- a/bistro/build/ci/make_docker_context.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import textwrap - -from build_steps import build_bistro -from fbcode_builder.make_docker_context import make_docker_context - -# Try --help for more info. -print(make_docker_context( - build_bistro, - help=textwrap.dedent(''' - - Creates a Docker context directory for building Bistro and its - dependencies, and prints its path to stdout. - - Pass --option facebook/bistro:git_hash SHA1 to build something other - than the master branch from Github. - - Or, pass --option facebook/bistro:local_repo_dir LOCAL_PATH to build - from a local repo instead of cloning from Github. - - Usage: - (cd $(./make_docker_context.py) && docker build . 2>&1 | tee log) - - '''), -)) diff --git a/bistro/build/.gitignore b/bistro/cmake/.gitignore similarity index 100% rename from bistro/build/.gitignore rename to bistro/cmake/.gitignore diff --git a/bistro/build/fbinclude/common/fb303/cpp/FacebookBase2.h b/bistro/cmake/fbinclude/common/fb303/cpp/FacebookBase2.h similarity index 93% rename from bistro/build/fbinclude/common/fb303/cpp/FacebookBase2.h rename to bistro/cmake/fbinclude/common/fb303/cpp/FacebookBase2.h index 32eb1f36a..18ea26ccb 100644 --- a/bistro/build/fbinclude/common/fb303/cpp/FacebookBase2.h +++ b/bistro/cmake/fbinclude/common/fb303/cpp/FacebookBase2.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, Facebook, Inc. + * Copyright (c) 2015-present, Facebook, Inc. * All rights reserved. * * This source code is licensed under the BSD-style license found in the diff --git a/bistro/build/fbinclude/common/fb303/if/fb303.thrift b/bistro/cmake/fbinclude/common/fb303/if/fb303.thrift similarity index 100% rename from bistro/build/fbinclude/common/fb303/if/fb303.thrift rename to bistro/cmake/fbinclude/common/fb303/if/fb303.thrift diff --git a/bistro/build/run-cmake.sh b/bistro/cmake/run-cmake.sh similarity index 92% rename from bistro/build/run-cmake.sh rename to bistro/cmake/run-cmake.sh index db6f363e5..b9708ecd0 100755 --- a/bistro/build/run-cmake.sh +++ b/bistro/cmake/run-cmake.sh @@ -9,7 +9,7 @@ # # -# Runs `cmake` for Bistro, directing output to build/Release/ or build/Debug/. +# Runs `cmake` for Bistro, directing output to cmake/Release/ or cmake/Debug/. # # If Bistro's dependencies are installed in a non-default prefix, it will # need some help finding them: @@ -87,10 +87,10 @@ fetch_gtest # Hacky way to build Thrift until we have a proper CMake module to do it. cd "$build_dir/.." -update_thrift if if/*.thrift build/fbinclude/common/fb303/if/fb303.thrift -./build/targets_to_cmake_lists.py . +update_thrift if if/*.thrift cmake/fbinclude/common/fb303/if/fb303.thrift +./cmake/targets_to_cmake_lists.py . -# Put our artifacts in build/{Debug,Release} +# Put our artifacts in cmake/{Debug,Release} make_dir="$build_dir/$build_type" mkdir -p "$make_dir" cd "$make_dir" diff --git a/bistro/build/setup.cmake b/bistro/cmake/setup.cmake similarity index 92% rename from bistro/build/setup.cmake rename to bistro/cmake/setup.cmake index ee2e4a0d6..5a8de8b7f 100644 --- a/bistro/build/setup.cmake +++ b/bistro/cmake/setup.cmake @@ -7,13 +7,13 @@ # of patent rights can be found in the PATENTS file in the same directory. # -# Do NOT use this -- run build/run-cmake.sh instead & read its docblock. +# Do NOT use this -- run cmake/run-cmake.sh instead & read its docblock. include_directories( # Our includes start with "bistro/bistro/" "${PROJECT_SOURCE_DIR}/../.." # A hack to include a stub for some FB-specific includes under "common/". - "${PROJECT_SOURCE_DIR}/build/fbinclude" + "${PROJECT_SOURCE_DIR}/cmake/fbinclude" "${CMAKE_INSTALL_PREFIX}/include" ) @@ -59,7 +59,7 @@ macro(bistro_link_libraries name) ) endmacro(bistro_link_libraries) -add_subdirectory(build/deps/gtest-1.7.0) +add_subdirectory(cmake/deps/gtest-1.7.0) enable_testing() include_directories("${gtest_SOURCE_DIR}/include" "${gtest_SOURCE_DIR}") diff --git a/bistro/build/targets_to_cmake_lists.py b/bistro/cmake/targets_to_cmake_lists.py similarity index 98% rename from bistro/build/targets_to_cmake_lists.py rename to bistro/cmake/targets_to_cmake_lists.py index 55a905f24..27e4068fb 100755 --- a/bistro/build/targets_to_cmake_lists.py +++ b/bistro/cmake/targets_to_cmake_lists.py @@ -111,7 +111,7 @@ def cpp_library(name, **kwargs): _validate_keys(kwargs, 'cpp_library', { 'srcs', # Handled above 'deps', # Handled above - 'external_deps', # Handled in build/CMakeLists.txt + 'external_deps', # Handled in cmake/setup.cmake 'headers', # CMake handles headers automatically }) @@ -195,5 +195,5 @@ def make_cmake_dir_recursive(root_path): make_cmake_dir_recursive(sys.argv[1]).write_cmake_lists( setup_lines='cmake_minimum_required(VERSION 2.8)\n' - 'include("build/setup.cmake")\n', + 'include("cmake/setup.cmake")\n', ) diff --git a/bistro/scripts/run_local_test.sh b/bistro/scripts/run_local_test.sh index 06e76a3de..67f6a24ca 100755 --- a/bistro/scripts/run_local_test.sh +++ b/bistro/scripts/run_local_test.sh @@ -30,9 +30,9 @@ if [[ -x "$BISTRO_BINARY" ]] ; then else PHABRICATOR_DOMAIN="[YOUR PHABRICATOR DOMAIN]" # Try the release binary, then default to the debug binary - BISTRO_BINARY="$D/../build/Release/bistro/bistro/server/bistro_scheduler" + BISTRO_BINARY="$D/../cmake/Release/bistro/bistro/server/bistro_scheduler" if [[ ! -x "$BISTRO_BINARY" ]] ; then - BISTRO_BINARY="$D/../build/Debug/bistro/bistro/server/bistro_scheduler" + BISTRO_BINARY="$D/../cmake/Debug/bistro/bistro/server/bistro_scheduler" fi fi diff --git a/bistro/build/ci/README.md b/build/README.md similarity index 100% rename from bistro/build/ci/README.md rename to build/README.md diff --git a/build/fbcode_builder/docker_builder.py b/build/fbcode_builder/docker_builder.py index f642f6cdb..b07c176b9 100644 --- a/build/fbcode_builder/docker_builder.py +++ b/build/fbcode_builder/docker_builder.py @@ -18,9 +18,11 @@ import os import tempfile -from .fbcode_builder import FBCodeBuilder -from .shell_quoting import raw_shell, shell_comment, shell_join, ShellQuoted -from .utils import recursively_flatten_list, run_command +from fbcode_builder import FBCodeBuilder +from shell_quoting import ( + raw_shell, shell_comment, shell_join, ShellQuoted +) +from utils import recursively_flatten_list, run_command class DockerFBCodeBuilder(FBCodeBuilder): diff --git a/build/fbcode_builder/fbcode_builder.py b/build/fbcode_builder/fbcode_builder.py index 1500a4985..b3e8e3689 100644 --- a/build/fbcode_builder/fbcode_builder.py +++ b/build/fbcode_builder/fbcode_builder.py @@ -37,9 +37,6 @@ shell script that runs outside of a container. Or maybe even synchronously executes the shell commands, `make`-style. - - Support for dependencies. This was not built since FB projects' - dependency chains are simple, so a linear sequence of steps was enough. - - A "Makefile" generator. That might make iterating on builds even quicker than what you can currently get with Docker build caching. @@ -47,7 +44,7 @@ import os -from .shell_quoting import path_join, shell_join, ShellQuoted +from shell_quoting import path_join, shell_join, ShellQuoted class FBCodeBuilder(object): @@ -99,7 +96,7 @@ def render(self, steps): raise RuntimeError( 'Unused options: {0} -- please check if you made a typo ' 'in any of them. Those that are truly not useful should ' - 'be removed so that this typo detection can be useful.' + 'be not be set so that this typo detection can be useful.' .format(unused_options) ) return res @@ -260,14 +257,6 @@ def make_and_install(self, make_vars=None): )), ] - def zstd_make_and_install(self): - 'Zstd is a special snowflake, so make it easy to built it correctly' - return self.step('Build and install zstd', [ - self.make_and_install(make_vars={ - 'PREFIX': self.option('prefix'), - }) - ]) - def autoconf_install(self, name): return self.step('Build and install {0}'.format(name), [ self.run(ShellQuoted('autoreconf -ivf')), diff --git a/build/fbcode_builder/fbcode_builder_config.py b/build/fbcode_builder/fbcode_builder_config.py new file mode 100644 index 000000000..eee11298b --- /dev/null +++ b/build/fbcode_builder/fbcode_builder_config.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +'Demo config, so that `make_docker_context.py --help` works in this directory.' + +config = { + 'fbcode_builder_spec': lambda _builder: { + 'depends_on': [], + 'steps': [], + }, + 'github_project': 'demo/project', +} diff --git a/build/fbcode_builder/make_docker_context.py b/build/fbcode_builder/make_docker_context.py old mode 100644 new mode 100755 index a6a760ce3..4e437d2b1 --- a/build/fbcode_builder/make_docker_context.py +++ b/build/fbcode_builder/make_docker_context.py @@ -4,32 +4,42 @@ from __future__ import print_function from __future__ import unicode_literals ''' +Reads `fbcode_builder_config.py` from the current directory, and prepares a +Docker context directory to build this project. Prints to stdout the path +to the context directory. -Helper for making a command-line utility that writes your project's -Dockerfile and associated data into a (temporary) directory. Your main -program will look something like this: - - print(make_docker_context( - lambda builder: [builder.step(...), ...], - help='Creates a Docker context and writes its path to stdout.', - )) +Try `.../make_docker_context.py --help` from a project's `build/` directory. +By default, the Docker context directory will be in /tmp. It will always +contain a Dockerfile, and might also contain copies of your local repos, and +other data needed for the build container. ''' import argparse import logging import os import tempfile +import textwrap -from fbcode_builder.docker_builder import DockerFBCodeBuilder -from .parse_args import parse_args_to_fbcode_builder_opts -from fbcode_builder.shell_quoting import raw_shell, ShellQuoted +from docker_builder import DockerFBCodeBuilder +from parse_args import parse_args_to_fbcode_builder_opts def make_docker_context( - get_steps_fn, opts=None, help='', default_context_dir=None + get_steps_fn, github_project, opts=None, default_context_dir=None ): - 'Returns a path to the Docker context directory. See parse_args.py.' + ''' + Returns a path to the Docker context directory. See parse_args.py. + + Helper for making a command-line utility that writes your project's + Dockerfile and associated data into a (temporary) directory. Your main + program might look something like this: + + print(make_docker_context( + lambda builder: [builder.step(...), ...], + 'facebook/your_project', + )) + ''' if opts is None: opts = {} @@ -77,6 +87,11 @@ def add_args(parser): help='Use `make -j` on multi-CPU systems with lots of RAM. ' 'Default: %(default)s.', ) + parser.add_argument( + '--local-repo-dir', metavar='DIR', + help='If set, build {0} from a local directory instead of Github.' + .format(github_project), + ) opts = parse_args_to_fbcode_builder_opts( add_args, @@ -89,11 +104,33 @@ def add_args(parser): 'os_image', 'gcc_version', 'make_parallelism', + 'local_repo_dir', ), opts, - help + help=textwrap.dedent(''' + + Reads `fbcode_builder_config.py` from the current directory, and + prepares a Docker context directory to build {github_project} and + its dependencies. Prints to stdout the path to the context + directory. + + Pass --option {github_project}:git_hash SHA1 to build something + other than the master branch from Github. + + Or, pass --option {github_project}:local_repo_dir LOCAL_PATH to + build from a local repo instead of cloning from Github. + + Usage: + (cd $(./make_docker_context.py) && docker build . 2>&1 | tee log) + + '''.format(github_project=github_project)), ) + # This allows travis_docker_build.sh not to know the main Github project. + local_repo_dir = opts.pop('local_repo_dir', None) + if local_repo_dir is not None: + opts['{0}:local_repo_dir'.format(github_project)] = local_repo_dir + if (opts.get('os_image'), opts.get('gcc_version')) not in valid_versions: raise Exception( 'Due to 4/5 ABI changes (std::string), we can only use {0}'.format( @@ -119,3 +156,14 @@ def add_args(parser): f.write(dockerfile) return context_dir + + +if __name__ == '__main__': + from utils import read_fbcode_builder_config, build_fbcode_builder_config + + # Load a spec from the current directory + config = read_fbcode_builder_config('fbcode_builder_config.py') + print(make_docker_context( + build_fbcode_builder_config(config), + config['github_project'], + )) diff --git a/build/fbcode_builder/parse_args.py b/build/fbcode_builder/parse_args.py index ac2eb4fb4..a47efafb5 100644 --- a/build/fbcode_builder/parse_args.py +++ b/build/fbcode_builder/parse_args.py @@ -8,7 +8,7 @@ import argparse import logging -from .shell_quoting import raw_shell, ShellQuoted +from shell_quoting import raw_shell, ShellQuoted def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help): diff --git a/build/fbcode_builder/__init__.py b/build/fbcode_builder/specs/__init__.py similarity index 100% rename from build/fbcode_builder/__init__.py rename to build/fbcode_builder/specs/__init__.py diff --git a/build/fbcode_builder/specs/fbthrift.py b/build/fbcode_builder/specs/fbthrift.py new file mode 100644 index 000000000..2c17dae37 --- /dev/null +++ b/build/fbcode_builder/specs/fbthrift.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.folly as folly +import specs.wangle as wangle +import specs.zstd as zstd + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + # This API should change rarely, so build the latest tag instead of master. + builder.add_option( + 'no1msd/mstch:git_hash', + ShellQuoted('$(git describe --abbrev=0 --tags)') + ) + return { + 'depends_on': [folly, wangle, zstd], + 'steps': [ + # This isn't a separete spec, since only fbthrift uses mstch. + builder.github_project_workdir('no1msd/mstch', '.'), + builder.cmake_install('no1msd/mstch'), + builder.fb_github_autoconf_install('fbthrift/thrift'), + ], + } diff --git a/build/fbcode_builder/specs/folly.py b/build/fbcode_builder/specs/folly.py new file mode 100644 index 000000000..64e94b93e --- /dev/null +++ b/build/fbcode_builder/specs/folly.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + return { + 'steps': [ + builder.fb_github_autoconf_install('folly/folly'), + ], + } diff --git a/build/fbcode_builder/specs/proxygen.py b/build/fbcode_builder/specs/proxygen.py new file mode 100644 index 000000000..6cdd8f33f --- /dev/null +++ b/build/fbcode_builder/specs/proxygen.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.folly as folly +import specs.wangle as wangle + + +def fbcode_builder_spec(builder): + return { + 'depends_on': [folly, wangle], + 'steps': [ + builder.fb_github_autoconf_install('proxygen/proxygen'), + ], + } diff --git a/build/fbcode_builder/specs/wangle.py b/build/fbcode_builder/specs/wangle.py new file mode 100644 index 000000000..57daba14a --- /dev/null +++ b/build/fbcode_builder/specs/wangle.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.folly as folly + + +def fbcode_builder_spec(builder): + # Projects that simply depend on Wangle need not spend time on tests. + builder.add_option('wangle/wangle:cmake_defines', {'BUILD_TESTS': 'OFF'}) + return { + 'depends_on': [folly], + 'steps': [ + builder.fb_github_cmake_install('wangle/wangle'), + ], + } diff --git a/build/fbcode_builder/specs/zstd.py b/build/fbcode_builder/specs/zstd.py new file mode 100644 index 000000000..c6a4e352d --- /dev/null +++ b/build/fbcode_builder/specs/zstd.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + # This API should change rarely, so build the latest tag instead of master. + builder.add_option( + 'facebook/zstd:git_hash', + ShellQuoted('$(git describe --abbrev=0 --tags)') + ) + return { + 'steps': [ + builder.github_project_workdir('facebook/zstd', '.'), + builder.step('Build and install zstd', [ + builder.make_and_install(make_vars={ + 'PREFIX': builder.option('prefix'), + }) + ]), + ], + } diff --git a/build/travis_docker_build.sh b/build/fbcode_builder/travis_docker_build.sh similarity index 69% rename from build/travis_docker_build.sh rename to build/fbcode_builder/travis_docker_build.sh index abcb6fda6..0bbdc0d80 100755 --- a/build/travis_docker_build.sh +++ b/build/fbcode_builder/travis_docker_build.sh @@ -6,13 +6,12 @@ gcc_version=${gcc_version?Must be set by Travis} make_parallelism=${make_parallelism:-4} cur_dir="$(readlink -f "$(dirname "$0")")" docker_context_dir=$( - # The PYTHONPATH will go away in phase 2 of simplifying fbcode_builder usage - PYTHONPATH="$cur_dir:$PYTHONPATH" \ - "$cur_dir/../bistro/build/ci/make_docker_context.py" \ + cd "$cur_dir/.." # Let the script find our fbcode_builder_config.py + "$cur_dir/make_docker_context.py" \ --os-image "$os_image" \ --gcc-version "$gcc_version" \ --make-parallelism "$make_parallelism" \ - --option facebook/bistro:local_repo_dir "$cur_dir/.." + --local-repo-dir "$cur_dir/../.." ) cd "${docker_context_dir?Failed to make Docker context directory}" docker build . diff --git a/build/fbcode_builder/utils.py b/build/fbcode_builder/utils.py index 3d460ac52..574f28a81 100644 --- a/build/fbcode_builder/utils.py +++ b/build/fbcode_builder/utils.py @@ -46,3 +46,40 @@ def push_dir(d): yield d finally: os.chdir(old_dir) + + +def read_fbcode_builder_config(filename): + # Allow one spec to read another + scope = {'read_fbcode_builder_config': read_fbcode_builder_config} + with open(filename) as config_file: + exec(config_file.read(), scope) + return scope['config'] + + +def steps_for_spec(builder, spec, processed_modules=None): + ''' + Sets `builder` configuration, and returns all the builder steps + necessary to build `spec` and its dependencies. + + Traverses the dependencies in depth-first order, honoring the sequencing + in each 'depends_on' list. + ''' + if processed_modules is None: + processed_modules = set() + steps = [] + for module in spec.get('depends_on', []): + if module not in processed_modules: + processed_modules.add(module) + steps.extend(steps_for_spec( + builder, + module.fbcode_builder_spec(builder), + processed_modules + )) + steps.extend(spec.get('steps', [])) + return steps + + +def build_fbcode_builder_config(config): + return lambda builder: builder.build( + steps_for_spec(builder, config['fbcode_builder_spec'](builder)) + ) diff --git a/build/fbcode_builder_config.py b/build/fbcode_builder_config.py new file mode 100644 index 000000000..9b41a45f1 --- /dev/null +++ b/build/fbcode_builder_config.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +'fbcode_builder steps to build & test Bistro' + +import specs.fbthrift as fbthrift +import specs.folly as folly +import specs.proxygen as proxygen + +from shell_quoting import ShellQuoted + + +# Since Bistro doesn't presently have an "install" target, there is no +# point in having its spec in the shared spec directory. +def fbcode_builder_spec(builder): + return { + 'depends_on': [folly, proxygen, fbthrift], + 'steps': [ + builder.fb_github_project_workdir('bistro/bistro'), + builder.step('Build bistro', [ + # Future: should this share some code with `cmake_install()`? + builder.run(ShellQuoted( + 'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages ' + './cmake/run-cmake.sh Debug -DCMAKE_INSTALL_PREFIX={p}' + ).format(p=builder.option('prefix'))), + builder.workdir('cmake/Debug'), + builder.parallel_make(), + ]), + builder.step('Run bistro tests', [ + builder.run(ShellQuoted('ctest')), + ]), + ] + } + + +config = { + 'github_project': 'facebook/bistro', + 'fbcode_builder_spec': fbcode_builder_spec, +}