Skip to content

Commit

Permalink
Use package to build Pants's wheels, rather than setup-py (Cherry…
Browse files Browse the repository at this point in the history
…-pick of #10947) (#10952)

[ci skip-rust]
  • Loading branch information
Eric-Arellano authored Oct 13, 2020
1 parent 969c8dc commit ee28d2c
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 43 deletions.
75 changes: 33 additions & 42 deletions build-support/bin/packages.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,12 @@
import shutil
import subprocess
import sys
from collections import defaultdict
from configparser import ConfigParser
from contextlib import contextmanager
from functools import total_ordering
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Dict, Iterable, Iterator, List, NamedTuple, Set, Tuple, cast
from typing import Dict, Iterable, Iterator, List, NamedTuple, Set, cast
from urllib.parse import quote_plus
from xml.etree import ElementTree

Expand All @@ -32,11 +31,9 @@ def __init__(
self,
name: str,
target: str,
bdist_wheel_flags: Tuple[str, ...] = ("--python-tag", "py36.py37.py38"),
) -> None:
self.name = name
self.target = target
self.bdist_wheel_flags = bdist_wheel_flags

def __lt__(self, other):
return self.name < other.name
Expand Down Expand Up @@ -84,7 +81,7 @@ def owners(self) -> Set[str]:
{
# NB: This a native wheel. We expect a distinct wheel for each Python version and each
# platform (macOS x linux).
Package("pantsbuild.pants", "src/python/pants:pants-packaged", bdist_wheel_flags=()),
Package("pantsbuild.pants", "src/python/pants:pants-packaged"),
Package("pantsbuild.pants.testutil", "src/python/pants/testutil:testutil_wheel"),
}
)
Expand Down Expand Up @@ -201,52 +198,46 @@ def build_pants_wheels() -> None:
destination = CONSTANTS.deploy_pants_wheel_dir / version
destination.mkdir(parents=True, exist_ok=True)

def build(packages: Iterable[Package], bdist_wheel_flags: Iterable[str]) -> None:
args = (
"./pants",
# TODO(#9924).
"--no-dynamic-ui",
# TODO(#7654): It's not safe to use Pantsd because we're already using Pants to run
# this script.
"--concurrent",
"setup-py",
*(package.target for package in packages),
"--",
"bdist_wheel",
*bdist_wheel_flags,
)
args = (
"./pants",
# TODO(#9924).
"--no-dynamic-ui",
# TODO(#7654): It's not safe to use Pantsd because we're already using Pants to run
# this script.
"--concurrent",
"package",
*(package.target for package in PACKAGES),
)

with set_pants_version(CONSTANTS.pants_unstable_version):
try:
subprocess.run(args, check=True)
for package in packages:
found_wheels = sorted(Path("dist").glob(f"{package}-{version}-*.whl"))
# NB: For any platform-specific wheels, like pantsbuild.pants, we assume that the
# top-level `dist` will only have wheels built for the current platform. This
# should be safe because it is not possible to build native wheels for another
# platform.
if not is_cross_platform(found_wheels) and len(found_wheels) > 1:
raise ValueError(
f"Found multiple wheels for {package} in the `dist/` folder, but was "
f"expecting only one wheel: {sorted(wheel.name for wheel in found_wheels)}."
)
for wheel in found_wheels:
if not (destination / wheel.name).exists():
# We use `copy2` to preserve metadata.
shutil.copy2(wheel, destination)
except subprocess.CalledProcessError as e:
failed_packages = ",".join(package.name for package in packages)
failed_targets = " ".join(package.target for package in packages)
failed_packages = ",".join(package.name for package in PACKAGES)
failed_targets = " ".join(package.target for package in PACKAGES)
die(
f"Failed to build packages {failed_packages} for {version} with targets "
f"{failed_targets}.\n\n{e!r}",
)

packages_by_flags = defaultdict(list)
for package in PACKAGES:
packages_by_flags[package.bdist_wheel_flags].append(package)
# TODO(#10718): Allow for sdist releases. We can build an sdist for
# `pantsbuild.pants.testutil`, but need to wire it up to the rest of our release process.
for package in PACKAGES:
found_wheels = sorted(Path("dist").glob(f"{package}-{version}-*.whl"))
# NB: For any platform-specific wheels, like pantsbuild.pants, we assume that the
# top-level `dist` will only have wheels built for the current platform. This
# should be safe because it is not possible to build native wheels for another
# platform.
if not is_cross_platform(found_wheels) and len(found_wheels) > 1:
raise ValueError(
f"Found multiple wheels for {package} in the `dist/` folder, but was "
f"expecting only one wheel: {sorted(wheel.name for wheel in found_wheels)}."
)
for wheel in found_wheels:
if not (destination / wheel.name).exists():
# We use `copy2` to preserve metadata.
shutil.copy2(wheel, destination)

with set_pants_version(CONSTANTS.pants_unstable_version):
for flags, packages in packages_by_flags.items():
build(packages, flags)
green(f"Wrote Pants wheels to {destination}.")


Expand Down
3 changes: 3 additions & 0 deletions src/python/pants/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ python_distribution(
':entry_point',
':version',
],
# Because we have native code, this will cause the wheel to use whatever the ABI is for the
# interpreter used to run setup.py, e.g. `cp36m-macosx_10_15_x86_64`.
setup_py_commands=["bdist_wheel"],
provides=setup_py(
name='pantsbuild.pants',
description='A scalable build tool for large, complex, heterogeneous repos.',
Expand Down
3 changes: 2 additions & 1 deletion src/python/pants/backend/python/goals/setup_py.py
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,7 @@ async def run_setup_py(
# TODO: Could there be other useful files to capture?
output_directories=(dist_dir,),
description=f"Run setuptools for {req.exported_target.target.address}",
level=LogLevel.DEBUG,
),
)
output_digest = await Get(Digest, RemovePrefix(result.output_digest, dist_dir))
Expand Down Expand Up @@ -739,7 +740,7 @@ async def get_requirements(
return ExportedTargetRequirements(req_strs)


@rule(desc="Find all code to be published in the distribution", level=LogLevel.INFO)
@rule(desc="Find all code to be published in the distribution", level=LogLevel.DEBUG)
async def get_owned_dependencies(
dependency_owner: DependencyOwner, union_membership: UnionMembership
) -> OwnedDependencies:
Expand Down
1 change: 1 addition & 0 deletions src/python/pants/testutil/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ python_distribution(
':testutil',
':int-test-for-export',
],
setup_py_commands=["bdist_wheel", "--python-tag", "py36.py37.py38", "sdist"],
provides=setup_py(
name='pantsbuild.pants.testutil',
description='Test support for writing Pants plugins.',
Expand Down

0 comments on commit ee28d2c

Please sign in to comment.