From 82e7b48665563c6e26c8306cfa583c7eb0475b24 Mon Sep 17 00:00:00 2001 From: Lukasz Piatkowski Date: Fri, 10 Jul 2020 12:04:28 -0700 Subject: [PATCH] edenscm/hg: add GitHub Actions with CI for HG plus add fixes for getdeps (#25) Summary: Fixes include: 1. Passing "GETDEPS_BUILD_DIR" and "GETDEPS_INSTALL_DIR" env variable and using them in eden/scm/Makefile rather than assuming the source code is always in the same place regardless getdeps arguments (it isn't). 2. Added "fbthrift-source" and "fb303-source" to avoid unnecessary compilation (at least of fb303) and to put fbthrift and fb303 source code in an easy to locate place inside getdeps' "installed" folder. Pull Request resolved: https://github.com/facebookexperimental/eden/pull/25 Test Plan: sandcastle, check oss-eden_scm-darwin-getdeps Reviewed By: farnz Differential Revision: D22431872 Pulled By: lukaspiatkowski fbshipit-source-id: 8ccbb090713ec085a5dd56df509eb58ab6fb9e34 --- build/fbcode_builder/getdeps/builder.py | 12 ------------ build/fbcode_builder/getdeps/buildopts.py | 15 +++++++++++++++ build/fbcode_builder/getdeps/runcmd.py | 12 ++++++------ build/fbcode_builder/manifests/eden_scm | 4 +++- build/fbcode_builder/manifests/fb303-source | 15 +++++++++++++++ .../manifests/{fbthrift-rust => fbthrift-source} | 2 +- build/fbcode_builder/manifests/mononoke | 2 +- 7 files changed, 41 insertions(+), 21 deletions(-) create mode 100644 build/fbcode_builder/manifests/fb303-source rename build/fbcode_builder/manifests/{fbthrift-rust => fbthrift-source} (94%) diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 1560375f6ac7..25c70b002785 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -990,7 +990,6 @@ def __init__( def run_cargo(self, install_dirs, operation, args=None): args = args or [] env = self._compute_env(install_dirs) - self.add_openssl_to_env(env, install_dirs) # Enable using nightly features with stable compiler env["RUSTC_BOOTSTRAP"] = "1" env["LIBZ_SYS_STATIC"] = "1" @@ -1002,17 +1001,6 @@ def run_cargo(self, install_dirs, operation, args=None): ] + args self._run_cmd(cmd, cwd=self.workspace_dir(), env=env) - def add_openssl_to_env(self, env, install_dirs): - openssl_candidates = [d for d in install_dirs if "openssl" in d] - if len(openssl_candidates) > 1: - raise Exception( - "Found more than one candidate for openssl directory: {}.".format( - openssl_candidates - ) - ) - elif len(openssl_candidates) == 1: - env["OPENSSL_DIR"] = openssl_candidates[0] - def build_source_dir(self): return os.path.join(self.build_dir, "source") diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py index 0d6a1b3079d2..9bc7e0cdbff8 100644 --- a/build/fbcode_builder/getdeps/buildopts.py +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -184,6 +184,9 @@ def compute_env_for_install_dirs(self, install_dirs, env=None, manifest=None): else: env = Env() + env["GETDEPS_BUILD_DIR"] = os.path.join(self.scratch_dir, "build") + env["GETDEPS_INSTALL_DIR"] = self.install_dir + if self.fbsource_dir: env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join( self.fbsource_dir, "xplat/third-party/yarn/offline-mirror" @@ -247,16 +250,28 @@ def compute_env_for_install_dirs(self, install_dirs, env=None, manifest=None): # If rustc is present in the `bin` directory, set RUSTC to prevent # cargo uses the rustc installed in the system. if self.is_windows(): + cargo_path = os.path.join(bindir, "cargo.bat") rustc_path = os.path.join(bindir, "rustc.bat") rustdoc_path = os.path.join(bindir, "rustdoc.bat") else: + cargo_path = os.path.join(bindir, "cargo") rustc_path = os.path.join(bindir, "rustc") rustdoc_path = os.path.join(bindir, "rustdoc") if os.path.isfile(rustc_path): + env["CARGO_BIN"] = cargo_path env["RUSTC"] = rustc_path env["RUSTDOC"] = rustdoc_path + if self.is_windows(): + libcrypto = os.path.join(d, "lib/libcrypto.lib") + else: + libcrypto = os.path.join(d, "lib/libcrypto.so") + openssl_include = os.path.join(d, "include/openssl") + if os.path.isfile(libcrypto) and os.path.isdir(openssl_include): + # This must be the openssl library, let Rust know about it + env["OPENSSL_DIR"] = d + return env diff --git a/build/fbcode_builder/getdeps/runcmd.py b/build/fbcode_builder/getdeps/runcmd.py index 89cfe395bc9d..b479a09e3f24 100644 --- a/build/fbcode_builder/getdeps/runcmd.py +++ b/build/fbcode_builder/getdeps/runcmd.py @@ -24,13 +24,13 @@ class RunCommandError(Exception): pass -def _print_env_diff(env): +def _print_env_diff(env, log_fn): current_keys = set(os.environ.keys()) wanted_env = set(env.keys()) unset_keys = current_keys.difference(wanted_env) for k in sorted(unset_keys): - print("+ unset %s" % k) + log_fn("+ unset %s\n" % k) added_keys = wanted_env.difference(current_keys) for k in wanted_env.intersection(current_keys): @@ -39,11 +39,11 @@ def _print_env_diff(env): for k in sorted(added_keys): if ("PATH" in k) and (os.pathsep in env[k]): - print("+ %s=\\" % k) + log_fn("+ %s=\\\n" % k) for elem in env[k].split(os.pathsep): - print("+ %s%s\\" % (shellquote(elem), os.pathsep)) + log_fn("+ %s%s\\\n" % (shellquote(elem), os.pathsep)) else: - print("+ %s=%s \\" % (k, shellquote(env[k]))) + log_fn("+ %s=%s \\\n" % (k, shellquote(env[k]))) def run_cmd(cmd, env=None, cwd=None, allow_fail=False, log_file=None): @@ -76,7 +76,7 @@ def _run_cmd(cmd, env, cwd, allow_fail, log_fn): if env: assert isinstance(env, Env) - _print_env_diff(env) + _print_env_diff(env, log_fn) # Convert from our Env type to a regular dict. # This is needed because python3 looks up b'PATH' and 'PATH' diff --git a/build/fbcode_builder/manifests/eden_scm b/build/fbcode_builder/manifests/eden_scm index a6e687cc715e..225e3723e378 100644 --- a/build/fbcode_builder/manifests/eden_scm +++ b/build/fbcode_builder/manifests/eden_scm @@ -43,8 +43,10 @@ fbcode/fboss/common = common \.pyc$ [dependencies] -fb303 +fb303-source fbthrift +fbthrift-source +openssl rust-shed [dependencies.fb=on] diff --git a/build/fbcode_builder/manifests/fb303-source b/build/fbcode_builder/manifests/fb303-source new file mode 100644 index 000000000000..ea160c50015a --- /dev/null +++ b/build/fbcode_builder/manifests/fb303-source @@ -0,0 +1,15 @@ +[manifest] +name = fb303-source +fbsource_path = fbcode/fb303 +shipit_project = fb303 +shipit_fbcode_builder = false + +[git] +repo_url = https://github.com/facebook/fb303.git + +[build] +builder = nop + +[shipit.pathmap] +fbcode/fb303/github = . +fbcode/fb303 = fb303 diff --git a/build/fbcode_builder/manifests/fbthrift-rust b/build/fbcode_builder/manifests/fbthrift-source similarity index 94% rename from build/fbcode_builder/manifests/fbthrift-rust rename to build/fbcode_builder/manifests/fbthrift-source index 95141a896715..72cfc3f107c3 100644 --- a/build/fbcode_builder/manifests/fbthrift-rust +++ b/build/fbcode_builder/manifests/fbthrift-source @@ -1,5 +1,5 @@ [manifest] -name = fbthrift-rust +name = fbthrift-source fbsource_path = fbcode/thrift shipit_project = fbthrift shipit_fbcode_builder = true diff --git a/build/fbcode_builder/manifests/mononoke b/build/fbcode_builder/manifests/mononoke index 8ee8e5210bea..9103a342898f 100644 --- a/build/fbcode_builder/manifests/mononoke +++ b/build/fbcode_builder/manifests/mononoke @@ -36,7 +36,7 @@ tools/rust/ossconfigs = . ^fbcode/configerator/structs/scm/mononoke/(?!public_autocargo).+/Cargo\.toml$ [dependencies] -fbthrift-rust +fbthrift-source rust-shed [dependencies.fb=on]