[facebook#38] Create build script generator

Currently the build_script 'build_openr.sh' is broken (facebook#38). Create a
debian system builder that creates a build_script from the
fbcode_builder. The shell_builder does something very similar. However,
the shell_builder doesn't install dependencies and programs to the
system, but to a temp directory.

The debian_system_builder creates a build script (like build_openr.sh)
that uses the fbcode_builder logic to install OpenR and its dependencies
to the system. Therefore, a build script created with the
debian_system_builder uses the same dependencies as the docker build
workflow that is integrated in the CI system.

Since the fbcode_builder is shared across multiple projects the
debian_system_builder lives in a separate directory. This requires some
hustle to import and reuse the fbcode_builder code (appending stuff to
the path).

To create a build_script run:

```
python debian_system_builder/debian_system_builder.py > ./build_openr.sh
sudo chmod +x build_openr.sh
sudo ./build_openr.sh
```

build/debian_system_builder/debian_specs/fbzmq.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# Copyright (c) Facebook, Inc. and its affiliates.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import specs.fbthrift as fbthrift
+import specs.folly as folly
+import specs.gmock as gmock
+import specs.sodium as sodium
+import specs.sigar as sigar
+
+from shell_quoting import ShellQuoted
+
+
+def fbcode_builder_spec(builder):
+    builder.add_option('zeromq/libzmq:git_hash', 'v4.2.5')
+    return {
+        'depends_on': [folly, fbthrift, gmock, sodium, sigar],
+        'steps': [
+            builder.github_project_workdir('zeromq/libzmq', '.'),
+            builder.step('Build and install zeromq/libzmq', [
+                builder.run(ShellQuoted('./autogen.sh')),
+                builder.configure(),
+                builder.make_and_install(),
+            ]),
+
+            builder.fb_github_project_workdir('fbzmq/fbzmq/build', 'facebook'),
+            builder.step('Build and install fbzmq/fbzmq/build', [
+                builder.cmake_configure('fbzmq/fbzmq/build'),
+                # we need the pythonpath to find the thrift compiler
+                builder.run(ShellQuoted(
+                    'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages '
+                    'make -j {n}'
+                ).format(p=builder.option('prefix'), n=builder.option('make_parallelism'))),
+                builder.run(ShellQuoted('sudo make install')),
+                builder.run(ShellQuoted('sudo ldconfig')),
+            ]),
+        ],
+    }

build/debian_system_builder/debian_system_builder.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python
+# Copyright (c) Facebook, Inc. and its affiliates.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+'''
+debian_system_builder.py allows running the fbcode_builder logic on the host
+rather than in a container and installs libraries and programs to the system.
+
+It emits a bash script with set -exo pipefail configured such that
+any failing step will cause the script to exit with failure.
+
+== How to run it? ==
+
+cd build
+python debian_system_builder/debian_system_builder.py > ./build_openr.sh
+sudo chmod +x build_openr.sh
+sudo ./build_openr.sh
+'''
+
+import os
+import distutils.spawn
+import fbcode_builder_path
+
+from fbcode_builder import FBCodeBuilder
+from shell_builder import ShellFBCodeBuilder
+from shell_quoting import (
+    raw_shell, shell_comment, shell_join, ShellQuoted, path_join
+)
+from utils import recursively_flatten_list
+
+
+class DebianSystemFBCodeBuilder(ShellFBCodeBuilder):
+
+    # Overwrite configure to remove prefix for system build
+    def configure(self, name=None):
+        autoconf_options = {}
+        if name is not None:
+            autoconf_options.update(
+                self.option('{0}:autoconf_options'.format(name), {})
+            )
+        return [
+            self.run(ShellQuoted(
+                'LDFLAGS="$LDFLAGS" '
+                'CFLAGS="$CFLAGS" '
+                'CPPFLAGS="$CPPFLAGS" '
+                './configure {args}'
+            ).format(
+                args=shell_join(' ', (
+                    ShellQuoted('{k}={v}').format(k=k, v=v)
+                    for k, v in autoconf_options.items()
+                )),
+            )),
+        ]
+
+    # Overwrite cmake_configure to remove prefix for system build
+    def cmake_configure(self, name, cmake_path='..'):
+        cmake_defines = {
+            'BUILD_SHARED_LIBS': 'ON'
+        }
+        cmake_defines.update(
+            self.option('{0}:cmake_defines'.format(name), {})
+        )
+        return [
+            self.run(ShellQuoted(
+                'CXXFLAGS="$CXXFLAGS -fPIC" '
+                'CFLAGS="$CFLAGS -fPIC" '
+                'cmake {args} {cmake_path}'
+            ).format(
+                args=shell_join(' ', (
+                    ShellQuoted('-D{k}={v}').format(k=k, v=v)
+                    for k, v in cmake_defines.items()
+                )),
+                cmake_path=cmake_path,
+            )),
+        ]
+
+    def github_project_workdir(self, project, path):
+        # Only check out a non-default branch if requested. This especially
+        # makes sense when building from a local repo.
+        git_hash = self.option(
+            '{0}:git_hash'.format(project),
+            # Any repo that has a hash in deps/github_hashes defaults to
+            # that, with the goal of making builds maximally consistent.
+            self._github_hashes.get(project, '')
+        )
+        maybe_change_branch = [
+            self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
+        ] if git_hash else []
+
+        base_dir = self.option('projects_dir')
+
+        local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
+        return self.step('Check out {0}, workdir {1}'.format(project, path), [
+            self.workdir(base_dir),
+            self.run(
+                ShellQuoted('if [[ ! -e "{p}" ]]; then \n'
+                            '\tgit clone https://github.com/{p}\n'
+                            'fi').format(p=project)
+            ) if not local_repo_dir else self.copy_local_repo(
+                local_repo_dir, os.path.basename(project)
+            ),
+            self.workdir(path_join(base_dir, os.path.basename(project), path)),
+        ] + maybe_change_branch)
+
+    # Cmake system install
+    def make_and_install(self, make_vars=None):
+        return [
+            self.parallel_make(make_vars),
+            self.run(ShellQuoted('sudo make install VERBOSE=1 {vars}').format(
+                vars=self._make_vars(make_vars)
+            )),
+            self.run(ShellQuoted('sudo ldconfig')),
+        ]
+
+    def setup(self):
+        steps = [
+            ShellQuoted('#!/bin/bash\n'),
+            ShellQuoted('set -exo pipefail'),
+            self.install_debian_deps(),
+        ]
+        if self.has_option('ccache_dir'):
+            ccache_dir = self.option('ccache_dir')
+            steps += [
+                ShellQuoted(
+                    # Set CCACHE_DIR before the `ccache` invocations below.
+                    'export CCACHE_DIR={ccache_dir} '
+                    'CC="ccache ${{CC:-gcc}}" CXX="ccache ${{CXX:-g++}}"'
+                ).format(ccache_dir=ccache_dir),
+            ]
+        return steps
+
+
+def install_dir():
+    install_dir = os.environ.get('INSTALL_DIR')
+    if not install_dir:
+        install_dir = '/usr/local'
+    return install_dir
+
+
+def ccache_dir():
+    ccache_dir = os.environ.get('CCACHE_DIR')
+    if not ccache_dir:
+        ccache_dir = '/ccache'
+    return ccache_dir
+
+
+def gcc_version():
+    gcc_version = os.environ.get('GCC_VERSION')
+    if not gcc_version:
+        gcc_version = '5'
+    return gcc_version
+
+
+if __name__ == '__main__':
+    from utils import read_fbcode_builder_config, build_fbcode_builder_config
+    install_dir = install_dir()
+
+    config_file = os.path.join(os.path.dirname(__file__),
+                               'debian_system_fbcode_builder_config.py')
+    config = read_fbcode_builder_config(config_file)
+    builder = DebianSystemFBCodeBuilder()
+
+    builder.add_option('projects_dir', install_dir)
+    if distutils.spawn.find_executable('ccache'):
+        ccache_dir = ccache_dir()
+        builder.add_option('ccache_dir', ccache_dir)
+    # Option is required by fbcode_builder_spec
+    builder.add_option('prefix', install_dir)
+    builder.add_option('make_parallelism', 4)
+    gcc_version = gcc_version()
+    builder.add_option('gcc_version', gcc_version)
+    make_steps = build_fbcode_builder_config(config)
+    steps = make_steps(builder)
+    print(builder.render(steps))

build/debian_system_builder/debian_system_fbcode_builder_config.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+
+#
+# Copyright (c) 2014-present, Facebook, Inc.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import specs.fbthrift as fbthrift
+import debian_specs.fbzmq as fbzmq
+import specs.folly as folly
+import specs.re2 as re2
+from shell_quoting import ShellQuoted, path_join
+
+
+"fbcode_builder steps to build & test Openr"
+
+
+def fbcode_builder_spec(builder):
+    builder.add_option("thom311/libnl:git_hash", "libnl3_2_25")
+    builder.add_option("openr/openr/build:cmake_defines", {"ADD_ROOT_TESTS": "OFF"})
+    maybe_curl_patch = []
+    patch = path_join(
+        builder.option("projects_dir"),
+        "../shipit_projects/openr/build/fix-route-obj-attr-list.patch",
+    )
+
+    if not builder.has_option("shipit_project_dir"):
+        maybe_curl_patch = [
+            builder.run(
+                ShellQuoted(
+                    "curl -O https://raw.githubusercontent.com/facebook/openr/master/"
+                    "build/fix-route-obj-attr-list.patch"
+                )
+            )
+        ]
+        patch = "fix-route-obj-attr-list.patch"
+    libnl_build_commands = maybe_curl_patch + [
+        builder.run(ShellQuoted("git apply {p}").format(p=patch)),
+        builder.run(ShellQuoted("./autogen.sh")),
+        builder.configure(),
+        builder.make_and_install(),
+    ]
+
+    return {
+        "depends_on": [folly, fbthrift, fbzmq, re2],
+        "steps": [
+            builder.github_project_workdir("thom311/libnl", "."),
+            builder.step("Build and install thom311/libnl", libnl_build_commands),
+            builder.fb_github_project_workdir("openr/openr/build", "facebook"),
+            builder.step(
+                "Build and install openr/openr/build",
+                [
+                    builder.cmake_configure("openr/openr/build"),
+                    # we need the pythonpath to find the thrift compiler
+                    builder.run(
+                        ShellQuoted(
+                            'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages '
+                            "make -j {n}"
+                        ).format(
+                            p=builder.option("prefix"),
+                            n=builder.option("make_parallelism"),
+                        )
+                    ),
+                    builder.run(ShellQuoted("sudo make install")),
+                    builder.run(ShellQuoted("sudo ldconfig")),
+                ],
+            ),
+            builder.step(
+                "Run openr tests",
+                [builder.run(ShellQuoted("CTEST_OUTPUT_ON_FAILURE=TRUE make test"))],
+            ),
+        ],
+    }
+
+
+config = {
+    "github_project": "facebook/openr",
+    "fbcode_builder_spec": fbcode_builder_spec,
+}

build/debian_system_builder/fbcode_builder_path.py

@@ -0,0 +1,8 @@
+import os
+import sys
+
+# Add fbcode_builder directory to the path
+sys.path.append(os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+    "fbcode_builder"
+))