Move fs_util building and release tagging to Python script (#9922)

This also fixes #9803. We now build `fs_util` in the wheel building shards, meaning that we will always release `fs_util`. We can do this because we are already paying the cost of Rust compilation due to #9881.
pantsbuild · May 31, 2020 · fc2d9e4 · fc2d9e4
1 parent 64495d8
commit fc2d9e4
Show file tree

Hide file tree

Showing 5 changed files with 91 additions and 64 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1597,7 +1597,6 @@ jobs:
     dist: xenial
     env:
     - CACHE_NAME=rust_tests.linux
-    - PREPARE_DEPLOY=1
     if: commit_message !~ /\[ci skip-rust-tests\]/
     name: Rust tests - Linux
     os: linux
@@ -1607,7 +1606,6 @@ jobs:
     - '3.7'
     script:
     - ./build-support/bin/ci.py --rust-tests
-    - ./build-support/bin/release.sh -f
     stage: Test Pants
     sudo: required
   - before_cache:
@@ -1636,14 +1634,12 @@ jobs:
     - PATH="${PYENV_ROOT}/versions/${PYENV_PY37_VERSION}/bin:${PATH}"
     - PATH="${PYENV_ROOT}/versions/${PYENV_PY38_VERSION}/bin:${PATH}"
     - CACHE_NAME=rust_tests.osx
-    - PREPARE_DEPLOY=1
     if: commit_message !~ /\[ci skip-rust-tests\]/
     name: Rust tests - OSX
     os: osx
     osx_image: xcode8
     script:
     - ./build-support/bin/ci.py --rust-tests
-    - ./build-support/bin/release.sh -f
     stage: Test Pants
   - addons:
       apt:
@@ -1691,7 +1687,7 @@ jobs:
     - PREPARE_DEPLOY=1
     - CACHE_NAME=wheels.linux
     language: python
-    name: Build Linux wheels
+    name: Build Linux wheels and fs_util
     os: linux
     python:
     - '2.7'
@@ -1703,7 +1699,8 @@ jobs:
       -g)" build-support/docker/travis_ci/
     - docker run --rm -t -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
       travis_ci:latest sh -c "./build-support/bin/release.sh -n && USE_PY37=true ./build-support/bin/release.sh
-      -n && USE_PY38=true ./build-support/bin/release.sh -n"
+      -n && USE_PY38=true ./build-support/bin/release.sh -n && ./build-support/bin/release.sh
+      -f"
     services:
     - docker
     stage: Test Pants
@@ -1742,13 +1739,14 @@ jobs:
     - PREPARE_DEPLOY=1
     - CACHE_NAME=wheels.osx
     language: generic
-    name: Build macOS wheels
+    name: Build macOS wheels and fs_util
     os: osx
     osx_image: xcode8
     script:
     - ./build-support/bin/release.sh -n
     - USE_PY37=true ./build-support/bin/release.sh -n
     - USE_PY38=true ./build-support/bin/release.sh -n
+    - ./build-support/bin/release.sh -f
     stage: Test Pants
   - before_install:
     - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64

diff --git a/build-support/bin/generate_travis_yml.py b/build-support/bin/generate_travis_yml.py
@@ -576,6 +576,8 @@ def _build_wheels_command() -> List[str]:
         "./build-support/bin/release.sh -n",
         "USE_PY37=true ./build-support/bin/release.sh -n",
         "USE_PY38=true ./build-support/bin/release.sh -n",
+        # NB: We also build `fs_util` in this shard to leverage having had compiled the engine.
+        "./build-support/bin/release.sh -f",
     ]
 
 
@@ -588,7 +590,7 @@ def build_wheels_linux() -> Dict:
     shard: Dict = {
         **CACHE_NATIVE_ENGINE,
         **linux_shard(use_docker=True),
-        "name": "Build Linux wheels",
+        "name": "Build Linux wheels and fs_util",
         "script": [docker_build_travis_ci_image(), docker_run_travis_ci_image(command)],
     }
     safe_extend(shard, "env", _build_wheels_env(platform=Platform.linux))
@@ -599,7 +601,7 @@ def build_wheels_osx() -> Dict:
     shard: Dict = {
         **CACHE_NATIVE_ENGINE,
         **osx_shard(osx_image="xcode8"),
-        "name": "Build macOS wheels",
+        "name": "Build macOS wheels and fs_util",
         "script": _build_wheels_command(),
         "before_install": _osx_before_install(
             python_versions=[PythonVersion.py36, PythonVersion.py37, PythonVersion.py38],
@@ -660,9 +662,7 @@ def integration_tests_v2(python_version: PythonVersion) -> Dict:
     **CACHE_NATIVE_ENGINE,
     "stage": Stage.test.value,
     "before_script": ["ulimit -n 8192"],
-    # NB: We also build `fs_util` in this shard to leverage having had compiled the engine. This
-    # requires setting PREPARE_DEPLOY=1.
-    "script": ["./build-support/bin/ci.py --rust-tests", "./build-support/bin/release.sh -f"],
+    "script": ["./build-support/bin/ci.py --rust-tests"],
     "if": SKIP_RUST_CONDITION,
 }
 
@@ -672,7 +672,7 @@ def rust_tests_linux() -> Dict:
         **_RUST_TESTS_BASE,
         **linux_fuse_shard(),
         "name": "Rust tests - Linux",
-        "env": ["CACHE_NAME=rust_tests.linux", "PREPARE_DEPLOY=1"],
+        "env": ["CACHE_NAME=rust_tests.linux"],
     }
 
 
@@ -701,7 +701,7 @@ def rust_tests_osx() -> Dict:
             # This is good, because `brew install openssl` would trigger the same issues as noted on why
             # we don't use the `addons` section.
         ],
-        "env": [*_osx_env_with_pyenv(), "CACHE_NAME=rust_tests.osx", "PREPARE_DEPLOY=1"],
+        "env": [*_osx_env_with_pyenv(), "CACHE_NAME=rust_tests.osx"],
     }
 
 

diff --git a/build-support/bin/packages.py b/build-support/bin/packages.py
@@ -19,7 +19,7 @@
 
 import requests
 from bs4 import BeautifulSoup
-from common import banner, die, green
+from common import banner, die, green, travis_section
 
 # -----------------------------------------------------------------------------------------------
 # Pants package definitions
@@ -228,6 +228,8 @@ def build(packages: Iterable[Package], bdist_wheel_flags: Iterable[str]) -> None
         formatted_flags = " ".join(bdist_wheel_flags)
         args = (
             "./v2",
+            # TODO(#9924).
+            "--no-dynamic-ui",
             # TODO(#7654): It's not safe to use Pantsd because we're already using Pants to run
             #  this script.
             "--concurrent",
@@ -270,6 +272,39 @@ def build(packages: Iterable[Package], bdist_wheel_flags: Iterable[str]) -> None
     green(f"Wrote Pants wheels to {destination}.")
 
 
+def build_fs_util() -> None:
+    # See https://pants.readme.io/docs/contributions-rust for a description of fs_util. We include
+    # it in our releases because it can be a useful standalone tool.
+    with travis_section("fs_util", "Building fs_util"):
+        subprocess.run(
+            [
+                "build-support/bin/native/cargo",
+                "build",
+                "--release",
+                "--manifest-path=src/rust/engine/Cargo.toml",
+                "-p",
+                "fs_util",
+            ],
+            check=True,
+            env={**os.environ, "RUST_BACKTRACE": "1"},
+        )
+        current_os = (
+            subprocess.run(["build-support/bin/get_os.sh"], stdout=subprocess.PIPE, check=True)
+            .stdout.decode()
+            .strip()
+        )
+        dest_dir = (
+            Path(CONSTANTS.deploy_dir)
+            / "bin"
+            / "fs_util"
+            / current_os
+            / CONSTANTS.pants_unstable_version
+        )
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        shutil.copy("src/rust/engine/target/release/fs_util", dest_dir)
+        green(f"Built fs_util at {dest_dir / 'fs_util'}.")
+
+
 def check_clean_git_branch() -> None:
     banner("Checking for a clean Git branch")
     git_status = (
@@ -447,15 +482,49 @@ def check_prebuilt_wheels(check_dir: str) -> None:
     green(f"All {len(all_packages())} pantsbuild.pants packages were fetched and are valid.")
 
 
+def tag_release() -> None:
+    tag_name = f"release_{CONSTANTS.pants_stable_version}"
+    subprocess.run(
+        [
+            "git",
+            "tag",
+            "-f",
+            f"--local-user={get_pgp_key_id()}",
+            "-m",
+            f"pantsbuild.pants release {CONSTANTS.pants_stable_version}",
+            tag_name,
+        ],
+        check=True,
+    )
+    subprocess.run(
+        ["git", "push", "-f", "git@github.com:pantsbuild/pants.git", tag_name], check=True
+    )
+
+
+def publish_docs_if_master_branch() -> None:
+    branch = get_git_branch()
+    if branch != "master":
+        print(
+            f"Skipping publish of pantsbuild.org because you are not on master ({branch}).\n\nTo "
+            "manually publish, first check out the `master` branch, then run "
+            "`build-support/bin/publish_docs.sh -p -y`."
+        )
+        return
+    subprocess.run(["build-support/bin/publish_docs.sh", "-p", "-y"], check=True)
+
+
 def create_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser()
     subparsers = parser.add_subparsers(dest="command")
 
     subparsers.add_parser("build-pants-wheels")
+    subparsers.add_parser("build-fs-util")
     subparsers.add_parser("check-release-prereqs")
     subparsers.add_parser("list-owners")
     subparsers.add_parser("list-packages")
     subparsers.add_parser("list-prebuilt-wheels")
+    # TODO: replace this with `publish-packages` once release.sh is ported.
+    subparsers.add_parser("post-publish")
 
     parser_fetch_prebuilt_wheels = subparsers.add_parser("fetch-and-check-prebuilt-wheels")
     parser_fetch_prebuilt_wheels.add_argument("--wheels-dest")
@@ -467,6 +536,8 @@ def main() -> None:
     args = create_parser().parse_args()
     if args.command == "build-pants-wheels":
         build_pants_wheels()
+    if args.command == "build-fs-util":
+        build_fs_util()
     if args.command == "check-release-prereqs":
         check_release_prereqs()
     if args.command == "list-owners":
@@ -475,6 +546,9 @@ def main() -> None:
         list_packages()
     if args.command == "list-prebuilt-wheels":
         list_prebuilt_wheels()
+    if args.command == "post-publish":
+        tag_release()
+        publish_docs_if_master_branch()
     if args.command == "fetch-and-check-prebuilt-wheels":
         with TemporaryDirectory() as tempdir:
             dest = args.wheels_dest or tempdir

diff --git a/build-support/bin/release.sh b/build-support/bin/release.sh
@@ -192,25 +192,6 @@ function build_3rdparty_packages() {
   end_travis_section
 }
 
-function build_fs_util() {
-  start_travis_section "fs_util" "Building fs_util binary"
-  # fs_util is a standalone tool which can be used to inspect and manipulate
-  # Pants's engine's file store, and interact with content addressable storage
-  # services which implement the Bazel remote execution API.
-  # It is a useful standalone tool which people may want to consume, for
-  # instance when debugging pants issues, or if they're implementing a remote
-  # execution API. Accordingly, we include it in our releases.
-  (
-    set -e
-    RUST_BACKTRACE=1 "${ROOT}/build-support/bin/native/cargo" build --release \
-      --manifest-path="${ROOT}/src/rust/engine/Cargo.toml" -p fs_util
-    dst_dir="${DEPLOY_DIR}/bin/fs_util/$("${ROOT}/build-support/bin/get_os.sh")/${PANTS_UNSTABLE_VERSION}"
-    mkdir -p "${dst_dir}"
-    cp "${ROOT}/src/rust/engine/target/release/fs_util" "${dst_dir}/"
-  ) || die "Failed to build fs_util"
-  end_travis_section
-}
-
 function activate_tmp_venv() {
   # Because the venv/bin/activate script's location is dynamic and not located in a fixed
   # place, Shellcheck will not be able to find it so we tell Shellcheck to ignore the file.
@@ -301,10 +282,6 @@ function dry_run_install() {
     -f "${DEPLOY_3RDPARTY_WHEEL_DIR}/${VERSION}" -f "${DEPLOY_PANTS_WHEEL_DIR}/${VERSION}"
 }
 
-function get_branch() {
-  git branch | grep -E '^\* ' | cut -d' ' -f2-
-}
-
 function get_pgp_keyid() {
   git config --get user.signingkey
 }
@@ -313,29 +290,6 @@ function get_pgp_program() {
   git config --get gpg.program || echo "gpg"
 }
 
-function check_release_prereqs() {
-  run_packages_script check-release-prereqs
-}
-
-function tag_release() {
-  release_version="${PANTS_STABLE_VERSION}" && \
-  tag_name="release_${release_version}" && \
-  git tag -f \
-    "--local-user=$(get_pgp_keyid)" \
-    -m "pantsbuild.pants release ${release_version}" \
-    "${tag_name}" && \
-  git push -f git@github.com:pantsbuild/pants.git "${tag_name}"
-}
-
-function publish_docs_if_master() {
-  branch=$(get_branch)
-  if [[ "${branch}" == "master" ]]; then
-    "${ROOT}/build-support/bin/publish_docs.sh" -p -y
-  else
-    echo "Skipping docsite publishing on non-master branch (${branch})."
-  fi
-}
-
 function reversion_whls() {
   # Reversions all whls from an input directory to an output directory.
   # Adds one pants-specific glob to match the `VERSION` file in `pantsbuild.pants`.
@@ -517,7 +471,7 @@ while getopts ":${_OPTS}" opt; do
     h) usage ;;
     d) debug="true" ;;
     n) dry_run="true" ;;
-    f) build_fs_util ; exit $? ;;
+    f) run_packages_script build-fs-util ; exit $? ;;
     t) test_release="true" ;;
     l) run_packages_script list-packages ; exit $? ;;
     o) run_packages_script list-owners ; exit $? ;;
@@ -551,7 +505,7 @@ elif [[ "${test_release}" == "true" ]]; then
 else
   banner "Releasing packages to PyPI"
   (
-    check_release_prereqs && publish_packages && tag_release && publish_docs_if_master && \
-      banner "Successfully released packages to PyPI"
+    run_packages_script check-release-prereqs && publish_packages && \
+    run_packages_script post-publish && banner "Successfully released packages to PyPI"
   ) || die "Failed to release packages to PyPI."
 fi
diff --git a/pants.travis-ci.toml b/pants.travis-ci.toml
@@ -11,6 +11,7 @@ travis_parallelism = 4
 [GLOBAL]
 # Override color support detection - we want colors and Travis supports them.
 colors = true
+# TODO: See #9924.
 dynamic_ui = false
 
 # TODO: Eventually we would like pantsd enabled in CI as well, but we blanket disable it for