Skip to content
Permalink
Browse files

Automatically clean old nightlies

Add a new `-keep` flag to `./mach clean-nightlies` to control how many
nightlies to keep, which keeps the n most recent nightlies found in git.

Use this to clean old nightlies at the start of each build,
currently keeping 3 nightlies at a time.
  • Loading branch information
aneeshusa committed Apr 18, 2017
1 parent b1824f7 commit 060a813b45f4b4a7eb854dbf288d25f67e8b7adf
Showing with 63 additions and 15 deletions.
  1. +17 −0 etc/ci/buildbot_steps.yml
  2. +46 −15 python/servo/bootstrap_commands.py
@@ -1,4 +1,5 @@
mac-rel-wpt1:
- ./mach clean-nightlies --keep 3 --force
- ./mach build --release
- ./mach test-wpt-failure
- ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
@@ -9,12 +10,14 @@ mac-rel-wpt1:
- bash ./etc/ci/manifest_changed.sh

mac-rel-wpt2:
- ./mach clean-nightlies --keep 3 --force
- ./mach build --release
- ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
- ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker
- ./mach build-geckolib --release

mac-dev-unit:
- ./mach clean-nightlies --keep 3 --force
- env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --dev
- env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach test-unit
- env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach package --dev
@@ -24,27 +27,32 @@ mac-dev-unit:
- bash ./etc/ci/manifest_changed.sh

mac-rel-css:
- ./mach clean-nightlies --keep 3 --force
- ./mach build --release
- ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed
- ./mach filter-intermittents css-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker
- bash ./etc/ci/lockfile_changed.sh
- bash ./etc/ci/manifest_changed.sh

mac-nightly:
- ./mach clean-nightlies --keep 3 --force
- ./mach build --release
- ./mach package --release
- ./etc/ci/upload_nightly.sh mac
- ./etc/ci/upload_nightly.sh macbrew

linux-rel-intermittent:
- ./mach clean-nightlies --keep 3 --force
- ./mach build --release
- ./etc/ci/check_intermittents.sh --log-raw intermittents.log

mac-rel-intermittent:
- ./mach clean-nightlies --keep 3 --force
- ./mach build --release
- ./etc/ci/check_intermittents.sh --log-raw intermittents.log

linux-dev:
- ./mach clean-nightlies --keep 3 --force
- ./mach test-tidy --no-progress --all
- ./mach test-tidy --no-progress --self-test
- env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --dev
@@ -59,13 +67,15 @@ linux-dev:
- bash ./etc/ci/check_no_panic.sh

linux-rel-wpt:
- ./mach clean-nightlies --keep 3 --force
- ./mach build --release --with-debug-assertions
- ./mach test-wpt-failure
- ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
- ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker
- ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource

linux-rel-css:
- ./mach clean-nightlies --keep 3 --force
- ./mach build --release --with-debug-assertions
- ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed
- ./mach filter-intermittents css-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker
@@ -76,39 +86,46 @@ linux-rel-css:
- bash ./etc/ci/manifest_changed.sh

linux-nightly:
- ./mach clean-nightlies --keep 3 --force
- ./mach build --release
- ./mach package --release
- ./etc/ci/upload_nightly.sh linux

android:
- ./mach clean-nightlies --keep 3 --force
- env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --android --dev
- env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach package --android --dev
- bash ./etc/ci/lockfile_changed.sh
- bash ./etc/ci/manifest_changed.sh
- python ./etc/ci/check_dynamic_symbols.py

android-nightly:
- ./mach clean-nightlies --keep 3 --force
- env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --android --release
- env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach package --android --release
- ./etc/ci/upload_nightly.sh android

arm32:
- ./mach clean-nightlies --keep 3 --force
- env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --rel --target=arm-unknown-linux-gnueabihf
- bash ./etc/ci/lockfile_changed.sh
- bash ./etc/ci/manifest_changed.sh

arm64:
- ./mach clean-nightlies --keep 3 --force
- env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --rel --target=aarch64-unknown-linux-gnu
- bash ./etc/ci/lockfile_changed.sh
- bash ./etc/ci/manifest_changed.sh

windows-msvc-dev:
- mach.bat clean-nightlies --keep 3 --force
- mach.bat build --dev
- mach.bat test-unit
- mach.bat package --dev
- mach.bat build-geckolib

windows-msvc-nightly:
- mach.bat clean-nightlies --keep 3 --force
- mach.bat build --release
- mach.bat package --release
- bash -l ./etc/ci/upload_nightly.sh windows-msvc
@@ -15,6 +15,7 @@
import os.path as path
import re
import shutil
import subprocess
import sys
import urllib2

@@ -26,7 +27,7 @@

import servo.bootstrap as bootstrap
from servo.command_base import CommandBase, BIN_SUFFIX
from servo.util import download_bytes, download_file, extract, host_triple
from servo.util import delete, download_bytes, download_file, extract, host_triple


@CommandProvider
@@ -285,26 +286,56 @@ def bootstrap_pub_suffix(self, force=False):
@CommandArgument('--force', '-f',
action='store_true',
help='Actually remove stuff')
def clean_nightlies(self, force=False):
rust_current = self.rust_path().split('/')[0]
@CommandArgument('--keep',
default='1',
help='Keep up to this many most recent nightlies')
def clean_nightlies(self, force=False, keep=None):
rust_current = self.rust_version()
cargo_current = self.cargo_build_id()
print("Current Rust version: " + rust_current)
print("Current Cargo version: " + cargo_current)
print("Current Rust version: {}".format(rust_current))
print("Current Cargo version: {}".format(cargo_current))
to_keep = {
'rust': set(),
'cargo': set(),
}
if int(keep) == 1:
# Optimize keep=1 case to not invoke git
to_keep['rust'].add(rust_current)
to_keep['cargo'].add(cargo_current)
else:
for tool in ["rust", "cargo"]:
commit_file = '{}-commit-hash'.format(tool)
cmd = subprocess.Popen(
['git', 'log', '--pretty=format:%H', '-n', keep, commit_file],
stdout=subprocess.PIPE,
universal_newlines=True
)
stdout, _ = cmd.communicate()
for commit in stdout.splitlines():
cmd = subprocess.Popen(
['git', 'show', '{}:{}'.format(commit, commit_file)],
stdout=subprocess.PIPE,
universal_newlines=True
)
commit_hash, _ = cmd.communicate()
to_keep[tool].add(commit_hash.rstrip())

removing_anything = False
for current, base in [(rust_current, "rust"), (cargo_current, "cargo")]:
base = path.join(self.context.sharedir, base)
for tool in ["rust", "cargo"]:
base = path.join(self.context.sharedir, tool)
for name in os.listdir(base):
if name != current:
# We append `-alt` if LLVM assertions aren't enabled,
# so use just the commit hash itself.
# This may occasionally leave an extra nightly behind
# but won't remove too many nightlies.
if name.partition('-')[0] not in to_keep[tool]:
removing_anything = True
name = path.join(base, name)
full_path = path.join(base, name)
if force:
print("Removing " + name)
if os.path.isdir(name):
shutil.rmtree(name)
else:
os.remove(name)
print("Removing {}".format(full_path))
delete(full_path)
else:
print("Would remove " + name)
print("Would remove {}".format(full_path))
if not removing_anything:
print("Nothing to remove.")
elif not force:

0 comments on commit 060a813

Please sign in to comment.
You can’t perform that action at this time.