Skip to content

Commit

Permalink
Use a single cpython clone.
Browse files Browse the repository at this point in the history
Having one clone per build was mandatory for parallel builds when we
used them. We no longer run multiple builds in paralllel, and now have
a lock to guarantee it'll never happen again.

Also having one clone per build takes 55GB on the server, and not reseting
between builds cause bugs like https://bugs.python.org/issue44006.
  • Loading branch information
JulienPalard committed Jun 6, 2021
1 parent 9bb60a3 commit b17bdc3
Showing 1 changed file with 5 additions and 8 deletions.
13 changes: 5 additions & 8 deletions build_docs.py
Expand Up @@ -189,12 +189,12 @@ def run(cmd) -> subprocess.CompletedProcess:
if result.returncode:
# Log last 20 lines, those are likely the interesting ones.
logging.error(
"Run KO: %r:\n%s",
"Run: %r KO:\n%s",
cmdstring,
indent("\n".join(result.stdout.split("\n")[-20:]), " "),
)
else:
logging.debug("Run OK: %r", cmdstring)
logging.debug("Run: %r OK", cmdstring)
result.check_returncode()
return result

Expand Down Expand Up @@ -228,16 +228,15 @@ def git_clone(repository, directory, branch=None):
raise AssertionError("Not a git repository.")
run(["git", "-C", directory, "fetch"])
if branch:
run(["git", "-C", directory, "checkout", branch])
run(["git", "-C", directory, "reset", "--hard", "origin/" + branch])
except (subprocess.CalledProcessError, AssertionError):
if os.path.exists(directory):
shutil.rmtree(directory)
logging.info("Cloning %s into %s", repository, directory)
os.makedirs(directory, mode=0o775)
run(["git", "clone", "--depth=1", "--no-single-branch", repository, directory])
run(["git", "clone", repository, directory])
if branch:
run(["git", "-C", directory, "checkout", branch])
run(["git", "-C", directory, "reset", "--hard", "origin/" + branch])


def version_to_tuple(version):
Expand Down Expand Up @@ -588,9 +587,7 @@ def lockfile(self):

@property
def checkout(self):
return os.path.join(
self.build_root, self.version.name, f"cpython-{self.language.tag}"
)
return os.path.join(self.build_root, "cpython")

def build(self):
logging.info(
Expand Down

0 comments on commit b17bdc3

Please sign in to comment.