Skip to content

Commit

Permalink
[internal] Port building release Pex to Python (#11958)
Browse files Browse the repository at this point in the history 
More progress on #11952. There is nothing left in Bash except for the processing of options, which then pipes to Python; and setting the Python interpreter to run the Python script. `release.sh` will be removed in a followup.

This adds lightweight validation to our PEX build process, which would have caught #11954.

Certainly, we should be building the PEX via `./pants package` instead of directly via PEX. But this is an incremental improvement from before.

[ci skip-rust]
  • Loading branch information
@Eric-Arellano
Eric-Arellano committed Apr 21, 2021
1 parent 009c54e commit f4c148b
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 140 deletions.
109 changes: 101 additions & 8 deletions build-support/bin/packages.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
from common import banner, die, green, travis_section
from reversion import reversion

from pants.util.strutil import strip_prefix

# -----------------------------------------------------------------------------------------------
# Pants package definitions
# -----------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -294,6 +296,31 @@ def create_twine_venv() -> None:
subprocess.run([CONSTANTS.twine_venv_dir / "bin/pip", "install", "--quiet", "twine"])


@contextmanager
def download_pex_bin() -> Iterator[Path]:
"""Download PEX and return the path to the binary."""
try:
pex_version = next(
strip_prefix(ln, "pex==").rstrip()
for ln in Path("3rdparty/python/requirements.txt").read_text().splitlines()
if ln.startswith("pex==")
)
except (FileNotFoundError, StopIteration) as exc:
die(
"Could not find a requirement starting with `pex==` in "
f"3rdparty/python/requirements.txt: {repr(exc)}"
)

with TemporaryDirectory() as tempdir:
resp = requests.get(
f"https://github.com/pantsbuild/pex/releases/download/v{pex_version}/pex"
)
resp.raise_for_status()
result = Path(tempdir, "pex")
result.write_bytes(resp.content)
yield result


# -----------------------------------------------------------------------------------------------
# Build artifacts
# -----------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -417,6 +444,71 @@ def build_fs_util() -> None:
green(f"Built fs_util at {dest_dir / 'fs_util'}.")


# TODO: We should be using `./pants package` and `pex_binary` for this...If Pants is lacking in
# capabilities, we should improve Pants. When porting, using `runtime_package_dependencies` to do
# the validation.
def build_pex(fetch: bool) -> None:
if fetch:
extra_pex_args = [
f"--platform={plat}-{abi}"
for plat in ("linux_x86_64", "macosx_10.15_x86_64")
for abi in ("cp-37-m", "cp-38-cp38", "cp-39-cp39")
]
pex_name = f"pants.{CONSTANTS.pants_unstable_version}.pex"
banner(f"Building {pex_name} by fetching wheels.")
else:
extra_pex_args = [f"--python={sys.executable}"]
plat = os.uname()[0].lower()
py = f"cp{''.join(map(str, sys.version_info[:2]))}"
pex_name = f"pants.{CONSTANTS.pants_unstable_version}.{plat}-{py}.pex"
banner(f"Building {pex_name} by building wheels.")

if CONSTANTS.deploy_dir.exists():
shutil.rmtree(CONSTANTS.deploy_dir)
CONSTANTS.deploy_dir.mkdir(parents=True)

if fetch:
fetch_prebuilt_wheels(CONSTANTS.deploy_dir)
check_prebuilt_wheels_present(CONSTANTS.deploy_dir)
else:
build_pants_wheels()
build_3rdparty_wheels()

dest = Path("dist") / pex_name
with download_pex_bin() as pex_bin:
subprocess.run(
[
sys.executable,
str(pex_bin),
"-o",
str(dest),
"--no-build",
"--no-pypi",
"--disable-cache",
"-f",
str(CONSTANTS.deploy_pants_wheel_dir / CONSTANTS.pants_unstable_version),
"-f",
str(CONSTANTS.deploy_3rdparty_wheel_dir / CONSTANTS.pants_unstable_version),
"--no-strip-pex-env",
"--console-script=pants",
"--unzip",
*extra_pex_args,
f"pantsbuild.pants=={CONSTANTS.pants_unstable_version}",
],
check=True,
)

if os.environ.get("PANTS_PEX_RELEASE", "") == "STABLE":
stable_dest = CONSTANTS.deploy_dir / "pex" / f"pants.{CONSTANTS.pants_stable_version}.pex"
stable_dest.parent.mkdir(parents=True, exist_ok=True)
dest.rename(stable_dest)
dest = stable_dest
green(f"Built {dest}")

subprocess.run([sys.executable, str(dest), "--version"], check=True)
green(f"Validated {dest}")


# -----------------------------------------------------------------------------------------------
# Publish
# -----------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -713,20 +805,18 @@ def check_prebuilt_wheels_present(check_dir: str | Path) -> None:
def create_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest="command")

subparsers.add_parser("publish")
subparsers.add_parser("dry-run-install")
subparsers.add_parser("test-release")
subparsers.add_parser("build-pants-wheels")
subparsers.add_parser("build-3rdparty-wheels")
subparsers.add_parser("build-fs-util")
subparsers.add_parser("build-local-pex")
subparsers.add_parser("build-universal-pex")
subparsers.add_parser("list-owners")
subparsers.add_parser("list-packages")
subparsers.add_parser("list-prebuilt-wheels")

parser_fetch_prebuilt_wheels = subparsers.add_parser("fetch-and-check-prebuilt-wheels")
parser_fetch_prebuilt_wheels.add_argument("--wheels-dest")

subparsers.add_parser("fetch-and-check-prebuilt-wheels")
return parser


Expand All @@ -744,6 +834,10 @@ def main() -> None:
build_3rdparty_wheels()
if args.command == "build-fs-util":
build_fs_util()
if args.command == "build-local-pex":
build_pex(fetch=False)
if args.command == "build-universal-pex":
build_pex(fetch=True)
if args.command == "list-owners":
list_owners()
if args.command == "list-packages":
Expand All @@ -752,9 +846,8 @@ def main() -> None:
list_prebuilt_wheels()
if args.command == "fetch-and-check-prebuilt-wheels":
with TemporaryDirectory() as tempdir:
dest = args.wheels_dest or tempdir
fetch_prebuilt_wheels(dest)
check_prebuilt_wheels_present(dest)
fetch_prebuilt_wheels(tempdir)
check_prebuilt_wheels_present(tempdir)


if __name__ == "__main__":
Expand Down
134 changes: 2 additions & 132 deletions build-support/bin/release.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,143 +36,13 @@ fi
# a temporary venv to build 3rdparty wheels.
export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['${interpreter_constraint}']"

# NB: Pants core does not have the ability to change its own version, so we compute the
# suffix here and mutate the VERSION_FILE to affect the current version.
readonly VERSION_FILE="${ROOT}/src/python/pants/VERSION"
PANTS_STABLE_VERSION="$(cat "${VERSION_FILE}")"
HEAD_SHA=$(git rev-parse --verify HEAD)
# We add a non-numeric prefix 'git' before the sha in order to avoid a hex sha which happens to
# contain only [0-9] being parsed as a number -- see #7399.
# TODO(#7399): mix in the timestamp before the sha instead of 'git' to get monotonic ordering!
readonly PANTS_UNSTABLE_VERSION="${PANTS_STABLE_VERSION}+git${HEAD_SHA:0:8}"

readonly DEPLOY_DIR="${ROOT}/dist/deploy"
readonly DEPLOY_3RDPARTY_WHEELS_PATH="wheels/3rdparty/${HEAD_SHA}"
readonly DEPLOY_PANTS_WHEELS_PATH="wheels/pantsbuild.pants/${HEAD_SHA}"
readonly DEPLOY_3RDPARTY_WHEEL_DIR="${DEPLOY_DIR}/${DEPLOY_3RDPARTY_WHEELS_PATH}"
readonly DEPLOY_PANTS_WHEEL_DIR="${DEPLOY_DIR}/${DEPLOY_PANTS_WHEELS_PATH}"

function run_packages_script() {
(
cd "${ROOT}"
./pants run "${ROOT}/build-support/bin/packages.py" -- "$@"
)
}

function safe_curl() {
real_curl="$(command -v curl)"
set +e
"${real_curl}" --fail -SL "$@"
exit_code=$?
set -e
if [[ "${exit_code}" -ne 0 ]]; then
echo >&2 "Curl failed with args: $*"
exit 1
fi
}

# A space-separated list of pants packages to include in any pexes that are built: by default,
# only pants core is included.
: "${PANTS_PEX_PACKAGES:="pantsbuild.pants"}"

# URL from which pex release binaries can be downloaded.
: "${PEX_DOWNLOAD_PREFIX:="https://github.com/pantsbuild/pex/releases/download"}"

function requirement() {
package="$1"
grep "^${package}[^A-Za-z0-9]" "${ROOT}/3rdparty/python/requirements.txt" || die "Could not find requirement for ${package}"
}

function run_pex() {
# TODO: Cache this in case we run pex multiple times
(
PEX_VERSION="$(requirement pex | sed -e "s|pex==||")"

pexdir="$(mktemp -d -t build_pex.XXXXX)"
trap 'rm -rf "${pexdir}"' EXIT

pex="${pexdir}/pex"

safe_curl -s "${PEX_DOWNLOAD_PREFIX}/v${PEX_VERSION}/pex" > "${pex}"
"${PY}" "${pex}" "$@"
)
}

function execute_pex() {
run_pex \
--no-build \
--no-pypi \
--disable-cache \
-f "${DEPLOY_PANTS_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
-f "${DEPLOY_3RDPARTY_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
"$@"
}

function build_pex() {
# Builds a pex from the current UNSTABLE version.
# If $1 == "build", builds a pex just for this platform, from source.
# If $1 == "fetch", fetches the linux and OSX wheels which were built on travis.
local mode="$1"

local linux_platform_noabi="linux_x86_64"
local osx_platform_noabi="macosx_10.15_x86_64"

case "${mode}" in
build)
# NB: When building locally, we explicitly target our local Py3. This will not be compatible
# with platforms other than `current` nor will it be compatible with multiple Python versions.
local distribution_target_flags=("--python=$(command -v "$PY")")
local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.${platform}.pex"
local stable_dest="${DEPLOY_DIR}/pex/pants.${PANTS_STABLE_VERSION}.pex"
;;
fetch)
local distribution_target_flags=()
abis=("cp-37-m" "cp-38-cp38" "cp-39-cp39")
for platform in "${linux_platform_noabi}" "${osx_platform_noabi}"; do
for abi in "${abis[@]}"; do
distribution_target_flags=("${distribution_target_flags[@]}" "--platform=${platform}-${abi}")
done
done
local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.pex"
local stable_dest="${DEPLOY_DIR}/pex/pants.${PANTS_STABLE_VERSION}.pex"
;;
*)
echo >&2 "Bad build_pex mode ${mode}"
exit 1
;;
esac

rm -rf "${DEPLOY_DIR}"
mkdir -p "${DEPLOY_DIR}"

if [[ "${mode}" == "fetch" ]]; then
run_packages_script fetch-and-check-prebuilt-wheels --wheels-dest "${DEPLOY_DIR}"
else
run_packages_script build-pants-wheels
run-packages-script build-3rdparty-wheels
fi

local requirements=()
for pkg_name in $PANTS_PEX_PACKAGES; do
requirements=("${requirements[@]}" "${pkg_name}==${PANTS_UNSTABLE_VERSION}")
done

execute_pex \
-o "${dest}" \
--no-strip-pex-env \
--script=pants \
--unzip \
"${distribution_target_flags[@]}" \
"${requirements[@]}"

if [[ "${PANTS_PEX_RELEASE}" == "stable" ]]; then
mkdir -p "$(dirname "${stable_dest}")"
cp "${dest}" "${stable_dest}"
fi

banner "Successfully built ${dest}"
}

_OPTS="hnftlowepq"

function usage() {
Expand Down Expand Up @@ -232,11 +102,11 @@ while getopts ":${_OPTS}" opt; do
exit $?
;;
p)
build_pex fetch
run_packages_script build-universal-pex
exit $?
;;
q)
build_pex build
run_packages_script build-local-pex
exit $?
;;
*) usage "Invalid option: -${OPTARG}" ;;
Expand Down

0 comments on commit f4c148b

Please sign in to comment.