feat: Add bootstrap-parallel command

.github/workflows/test.yaml

@@ -85,6 +85,7 @@ jobs:
           - bootstrap_sdist_only
           - bootstrap_git_url
           - bootstrap_git_url_tag
+          - bootstrap_parallel
           - bootstrap_skip_constraints
           - build
           - build_order

.mergify.yml

@@ -45,6 +45,7 @@ pull_request_rules:
           - check-success=e2e (3.11, 1.75, bootstrap_extras, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, bootstrap_git_url, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, bootstrap_git_url_tag, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, bootstrap_parallel, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, bootstrap_prerelease, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, bootstrap_sdist_only, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, bootstrap_skip_constraints, ubuntu-latest)
@@ -85,6 +86,8 @@ pull_request_rules:
           - check-success=e2e (3.12, 1.75, bootstrap_git_url, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_git_url_tag, macos-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_git_url_tag, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_parallel, macos-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_parallel, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_prerelease, macos-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_prerelease, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_sdist_only, macos-latest)

e2e/test_bootstrap_parallel.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# Tests bootstrap parallel (bootstrap --sdist-only + build-parallel)
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+
+# passing settings to bootstrap but should have 0 effect on it
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --settings-dir="$SCRIPTDIR/changelog_settings" \
+  bootstrap-parallel 'stevedore==5.2.0'
+
+find "$OUTDIR/wheels-repo/" -name '*.whl'
+find "$OUTDIR/sdists-repo/" -name '*.tar.gz'
+ls "$OUTDIR"/work-dir/*/build.log || true
+
+EXPECTED_FILES="
+$OUTDIR/wheels-repo/downloads/setuptools-*.whl
+$OUTDIR/wheels-repo/downloads/pbr-*.whl
+$OUTDIR/wheels-repo/downloads/stevedore-*.whl
+
+$OUTDIR/sdists-repo/downloads/stevedore-*.tar.gz
+$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
+$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
+
+$OUTDIR/sdists-repo/builds/stevedore-*.tar.gz
+$OUTDIR/sdists-repo/builds/setuptools-*.tar.gz
+$OUTDIR/sdists-repo/builds/pbr-*.tar.gz
+
+$OUTDIR/work-dir/build-order.json
+$OUTDIR/work-dir/constraints.txt
+
+$OUTDIR/bootstrap.log
+$OUTDIR/fromager-errors.log
+
+$OUTDIR/work-dir/pbr-*/build.log
+$OUTDIR/work-dir/setuptools-*/build.log
+$OUTDIR/work-dir/stevedore-*/build.log
+"
+
+pass=true
+for pattern in $EXPECTED_FILES; do
+  if [ ! -f "${pattern}" ]; then
+    echo "Did not find $pattern" 1>&2
+    pass=false
+  fi
+done
+
+$pass

src/fromager/bootstrapper.py

@@ -309,7 +309,7 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version:
 
         # we are done processing this req, so lets remove it from the why chain
         self.why.pop()
-        self._cleanup(req, sdist_root_dir, build_env)
+        self.ctx.clean_build_dirs(sdist_root_dir, build_env)
         return resolved_version
 
     @property
@@ -877,26 +877,6 @@ def _create_unpack_dir(self, req: Requirement, resolved_version: Version):
         unpack_dir.mkdir(parents=True, exist_ok=True)
         return unpack_dir
 
-    def _cleanup(
-        self,
-        req: Requirement,
-        sdist_root_dir: pathlib.Path | None,
-        build_env: build_environment.BuildEnvironment | None,
-    ) -> None:
-        if not self.ctx.cleanup:
-            return
-
-        # Cleanup the source tree and build environment, leaving any other
-        # artifacts that were created.
-        if sdist_root_dir and sdist_root_dir.exists():
-            logger.debug(f"cleaning up source tree {sdist_root_dir}")
-            shutil.rmtree(sdist_root_dir)
-            logger.debug(f"cleaned up source tree {sdist_root_dir}")
-        if build_env and build_env.path.exists():
-            logger.debug(f"cleaning up build environment {build_env.path}")
-            shutil.rmtree(build_env.path)
-            logger.debug(f"cleaned up build environment {build_env.path}")
-
     def _add_to_graph(
         self,
         req: Requirement,

src/fromager/commands/__init__.py

@@ -17,6 +17,7 @@
 
 commands = [
     bootstrap.bootstrap,
+    bootstrap.bootstrap_parallel,
     build.build,
     build.build_sequence,
     build.build_parallel,

src/fromager/commands/bootstrap.py

@@ -1,5 +1,7 @@
 import logging
+import time
 import typing
+from datetime import timedelta
 
 import click
 from packaging.requirements import Requirement
@@ -19,6 +21,7 @@
 )
 from ..log import requirement_ctxvar
 from ..requirements_file import RequirementType
+from .build import build_parallel
 from .graph import find_why
 
 # Map child_name==child_version to list of (parent_name==parent_version, Requirement)
@@ -402,3 +405,118 @@ def write_constraints_file(
 
 
 bootstrap._fromager_show_build_settings = True  # type: ignore
+
+
+@click.command()
+@click.option(
+    "-r",
+    "--requirements-file",
+    "requirements_files",
+    multiple=True,
+    type=str,
+    help="pip requirements file",
+)
+@click.option(
+    "-p",
+    "--previous-bootstrap-file",
+    "previous_bootstrap_file",
+    type=str,
+    help="graph file produced from a previous bootstrap",
+)
+@click.option(
+    "-c",
+    "--cache-wheel-server-url",
+    "cache_wheel_server_url",
+    help="url to a wheel server from where fromager can download the wheels that it has built before",
+)
+@click.option(
+    "--skip-constraints",
+    "skip_constraints",
+    is_flag=True,
+    default=False,
+    help="Skip generating constraints.txt file to allow building collections with conflicting versions",
+)
+@click.option(
+    "-f",
+    "--force",
+    is_flag=True,
+    default=False,
+    help="rebuild wheels even if they have already been built",
+)
+@click.option(
+    "-m",
+    "--max-workers",
+    type=int,
+    default=None,
+    help="maximum number of parallel workers to run (default: unlimited)",
+)
+@click.argument("toplevel", nargs=-1)
+@click.pass_obj
+@click.pass_context
+def bootstrap_parallel(
+    ctx: click.Context,
+    wkctx: context.WorkContext,
+    *,
+    requirements_files: list[str],
+    previous_bootstrap_file: str | None,
+    cache_wheel_server_url: str | None,
+    skip_constraints: bool,
+    force: bool,
+    max_workers: int | None,
+    toplevel: list[str],
+) -> None:
+    """Bootstrap and build-parallel
+
+    Bootstraps all dependencies in sdist-only mode, then builds the
+    remaining wheels in parallel. The bootstrap step downloads sdists
+    and builds build-time dependency in serial. The build-parallel step
+    builds the remaining wheels in parallel.
+    """
+    # Do not remove build environments in bootstrap phase to speed up the
+    # parallel build phase.
+    logger.info("keep build env for build-parallel phase")
+    wkctx.cleanup_buildenv = False
+
+    start = time.perf_counter()
+    logger.info("*** starting bootstrap in sdist-only mode ***")
+    ctx.invoke(
+        bootstrap,
+        requirements_files=requirements_files,
+        previous_bootstrap_file=previous_bootstrap_file,
+        cache_wheel_server_url=cache_wheel_server_url,
+        sdist_only=True,
+        skip_constraints=skip_constraints,
+        toplevel=toplevel,
+    )
+
+    # statistics
+    wheels = sorted(f.name for f in wkctx.wheels_downloads.glob("*.whl"))
+    sdists = sorted(f.name for f in wkctx.sdists_downloads.glob("*.tar.gz"))
+    logger.debug("wheels: %s", ", ".join(wheels))
+    logger.debug("sdists: %s", ", ".join(sdists))
+    logger.info("bootstrap: %i wheels, %i sdists", len(wheels), len(sdists))
+    logger.info(
+        "*** finished bootstrap in %s ***\n",
+        timedelta(seconds=round(time.perf_counter() - start, 0)),
+    )
+
+    # reset dependency graph
+    wkctx.dependency_graph.clear()
+
+    # cleanup build envs in build-parallel step
+    wkctx.cleanup_buildenv = wkctx.cleanup
+
+    start_build = time.perf_counter()
+    logger.info("*** starting build-parallel with %s ***", wkctx.graph_file)
+    ctx.invoke(
+        build_parallel,
+        cache_wheel_server_url=cache_wheel_server_url,
+        max_workers=max_workers,
+        force=force,
+        graph_file=wkctx.graph_file,
+    )
+    logger.info(
+        "*** finished build-parallel in %s, total %s ***\n",
+        timedelta(seconds=round(time.perf_counter() - start_build, 0)),
+        timedelta(seconds=round(time.perf_counter() - start, 0)),
+    )

src/fromager/commands/build.py

@@ -438,6 +438,8 @@ def _build(
             wheel_filename=wheel_filename,
         )
 
+        wkctx.clean_build_dirs(source_root_dir, build_env)
+
     root_logger.removeHandler(file_handler)
     file_handler.close()
 

src/fromager/context.py

@@ -1,8 +1,12 @@
+from __future__ import annotations
+
 import collections
 import logging
 import os
 import pathlib
+import shutil
 import threading
+import typing
 from urllib.parse import urlparse
 
 from packaging.requirements import Requirement
@@ -11,6 +15,9 @@
 
 from . import constraints, dependency_graph, packagesettings, request_session
 
+if typing.TYPE_CHECKING:
+    from . import build_environment
+
 logger = logging.getLogger(__name__)
 
 # Map package names to (requirement type, dependency name, version)
@@ -58,9 +65,12 @@ def __init__(
         self.wheels_prebuilt = self.wheels_repo / "prebuilt"
         self.wheel_server_dir = self.wheels_repo / "simple"
         self.work_dir = pathlib.Path(work_dir).absolute()
+        self.graph_file = self.work_dir / "graph.json"
         self.wheel_server_url = ""
         self.logs_dir = self.work_dir / "logs"
         self.cleanup = cleanup
+        # separate value so bootstrap-parallel can keep build envs
+        self.cleanup_buildenv = cleanup
         self.variant = variant
         self.network_isolation = network_isolation
         self.settings_dir = settings_dir
@@ -116,7 +126,7 @@ def pip_constraint_args(self) -> list[str]:
         return ["--constraint", os.fspath(path_to_constraints_file)]
 
     def write_to_graph_to_file(self):
-        with open(self.work_dir / "graph.json", "w") as f:
+        with self.graph_file.open("w", encoding="utf-8") as f:
             self.dependency_graph.serialize(f)
 
     def package_build_info(
@@ -146,3 +156,31 @@ def setup(self) -> None:
             if not p.exists():
                 logger.debug("creating %s", p)
                 p.mkdir(parents=True)
+
+    def clean_build_dirs(
+        self,
+        sdist_root_dir: pathlib.Path | None,
+        build_env: build_environment.BuildEnvironment | None,
+    ) -> None:
+        """Cleanup the source tree and build environment
+
+        Leaving any other artifacts that were created.
+        """
+        if sdist_root_dir and build_env and build_env.path.parent == sdist_root_dir:
+            raise ValueError(f"Invalud {sdist_root_dir}, parent of {build_env}")
+
+        if sdist_root_dir and sdist_root_dir.exists():
+            if self.cleanup:
+                logger.debug(f"cleaning up source tree {sdist_root_dir}")
+                shutil.rmtree(sdist_root_dir)
+                logger.debug(f"cleaned up source tree {sdist_root_dir}")
+            else:
+                logger.debug(f"keeping source tree {sdist_root_dir}")
+
+        if build_env and build_env.path.exists():
+            if self.cleanup_buildenv:
+                logger.debug(f"cleaning up build environment {build_env.path}")
+                shutil.rmtree(build_env.path)
+                logger.debug(f"cleaned up build environment {build_env.path}")
+            else:
+                logger.debug(f"keeping build environment {build_env.path}")

src/fromager/dependency_graph.py

@@ -117,8 +117,7 @@ def to_dict(self) -> DependencyEdgeDict:
 class DependencyGraph:
     def __init__(self) -> None:
         self.nodes: dict[str, DependencyNode] = {}
-        root = DependencyNode.construct_root_node()
-        self.nodes[ROOT] = root
+        self.clear()
 
     @classmethod
     def from_file(
@@ -165,6 +164,10 @@ def from_dict(
             visited.add(curr_key)
         return graph
 
+    def clear(self) -> None:
+        self.nodes.clear()
+        self.nodes[ROOT] = DependencyNode.construct_root_node()
+
     def _to_dict(self):
         raw_graph = {}
         stack = [self.nodes[ROOT]]

tests/test_commands.py

@@ -0,0 +1,21 @@
+import typing
+
+import click
+
+from fromager.commands import bootstrap, build
+
+
+def get_option_names(cmd: click.Command) -> typing.Iterable[str]:
+    return [o.name for o in cmd.params if o.name]
+
+
+def test_bootstrap_pallel_options() -> None:
+    expected: set[str] = set()
+    expected.update(get_option_names(bootstrap.bootstrap))
+    expected.update(get_option_names(build.build_parallel))
+    # bootstrap-parallel enforces sdist_only=True and handles
+    # graph_file internally.
+    expected.discard("sdist_only")
+    expected.discard("graph_file")
+
+    assert set(get_option_names(bootstrap.bootstrap_parallel)) == expected