Auto merge of #16487 - aneeshusa:automatically-clean-old-nightlies, r…

…=larsbergstrom

Automatically clean old nightlies

<!-- Please describe your changes on the following line: -->

Automatically clean old rustc and cargo nightlies on our builders up to preserve disk space.

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [ ] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [ ] These changes fix #12103 and help with servo/saltfs#321 (github issue number if applicable).

<!-- Either: -->
- [ ] There are tests for these changes OR
- [x] These changes do not require tests because tested manually

<!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.-->

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/16487)
<!-- Reviewable:end -->

etc/ci/buildbot_steps.yml

@@ -1,4 +1,5 @@
 mac-rel-wpt1:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release
   - ./mach test-wpt-failure
   - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
@@ -9,12 +10,14 @@ mac-rel-wpt1:
   - bash ./etc/ci/manifest_changed.sh
 
 mac-rel-wpt2:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release
   - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
   - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker
   - ./mach build-geckolib --release
 
 mac-dev-unit:
+  - ./mach clean-nightlies --keep 3 --force
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --dev
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach test-unit
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach package --dev
@@ -24,27 +27,32 @@ mac-dev-unit:
   - bash ./etc/ci/manifest_changed.sh
 
 mac-rel-css:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release
   - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed
   - ./mach filter-intermittents css-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker
   - bash ./etc/ci/lockfile_changed.sh
   - bash ./etc/ci/manifest_changed.sh
 
 mac-nightly:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release
   - ./mach package --release
   - ./etc/ci/upload_nightly.sh mac
   - ./etc/ci/upload_nightly.sh macbrew
 
 linux-rel-intermittent:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release
   - ./etc/ci/check_intermittents.sh --log-raw intermittents.log
 
 mac-rel-intermittent:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release
   - ./etc/ci/check_intermittents.sh --log-raw intermittents.log
 
 linux-dev:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach test-tidy --no-progress --all
   - ./mach test-tidy --no-progress --self-test
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --dev
@@ -59,13 +67,15 @@ linux-dev:
   - bash ./etc/ci/check_no_panic.sh
 
 linux-rel-wpt:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release --with-debug-assertions
   - ./mach test-wpt-failure
   - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
   - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker
   - ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource
 
 linux-rel-css:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release --with-debug-assertions
   - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed
   - ./mach filter-intermittents css-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker
@@ -76,39 +86,46 @@ linux-rel-css:
   - bash ./etc/ci/manifest_changed.sh
 
 linux-nightly:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release
   - ./mach package --release
   - ./etc/ci/upload_nightly.sh linux
 
 android:
+  - ./mach clean-nightlies --keep 3 --force
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --android --dev
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach package --android --dev
   - bash ./etc/ci/lockfile_changed.sh
   - bash ./etc/ci/manifest_changed.sh
   - python ./etc/ci/check_dynamic_symbols.py
 
 android-nightly:
+  - ./mach clean-nightlies --keep 3 --force
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --android --release
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach package --android --release
   - ./etc/ci/upload_nightly.sh android
 
 arm32:
+  - ./mach clean-nightlies --keep 3 --force
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --rel --target=arm-unknown-linux-gnueabihf
   - bash ./etc/ci/lockfile_changed.sh
   - bash ./etc/ci/manifest_changed.sh
 
 arm64:
+  - ./mach clean-nightlies --keep 3 --force
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --rel --target=aarch64-unknown-linux-gnu
   - bash ./etc/ci/lockfile_changed.sh
   - bash ./etc/ci/manifest_changed.sh
 
 windows-msvc-dev:
+  - mach.bat clean-nightlies --keep 3 --force
   - mach.bat build --dev
   - mach.bat test-unit
   - mach.bat package --dev
   - mach.bat build-geckolib
 
 windows-msvc-nightly:
+  - mach.bat clean-nightlies --keep 3 --force
   - mach.bat build --release
   - mach.bat package --release
   - bash -l ./etc/ci/upload_nightly.sh windows-msvc

python/servo/bootstrap_commands.py

@@ -15,6 +15,7 @@
 import os.path as path
 import re
 import shutil
+import subprocess
 import sys
 import urllib2
 
@@ -26,7 +27,7 @@
 
 import servo.bootstrap as bootstrap
 from servo.command_base import CommandBase, BIN_SUFFIX
-from servo.util import download_bytes, download_file, extract, host_triple
+from servo.util import delete, download_bytes, download_file, extract, host_triple
 
 
 @CommandProvider
@@ -285,26 +286,56 @@ def bootstrap_pub_suffix(self, force=False):
     @CommandArgument('--force', '-f',
                      action='store_true',
                      help='Actually remove stuff')
-    def clean_nightlies(self, force=False):
-        rust_current = self.rust_path().split('/')[0]
+    @CommandArgument('--keep',
+                     default='1',
+                     help='Keep up to this many most recent nightlies')
+    def clean_nightlies(self, force=False, keep=None):
+        rust_current = self.rust_version()
         cargo_current = self.cargo_build_id()
-        print("Current Rust version: " + rust_current)
-        print("Current Cargo version: " + cargo_current)
+        print("Current Rust version: {}".format(rust_current))
+        print("Current Cargo version: {}".format(cargo_current))
+        to_keep = {
+            'rust': set(),
+            'cargo': set(),
+        }
+        if int(keep) == 1:
+            # Optimize keep=1 case to not invoke git
+            to_keep['rust'].add(rust_current)
+            to_keep['cargo'].add(cargo_current)
+        else:
+            for tool in ["rust", "cargo"]:
+                commit_file = '{}-commit-hash'.format(tool)
+                cmd = subprocess.Popen(
+                    ['git', 'log', '--pretty=format:%H', '-n', keep, commit_file],
+                    stdout=subprocess.PIPE,
+                    universal_newlines=True
+                )
+                stdout, _ = cmd.communicate()
+                for commit in stdout.splitlines():
+                    cmd = subprocess.Popen(
+                        ['git', 'show', '{}:{}'.format(commit, commit_file)],
+                        stdout=subprocess.PIPE,
+                        universal_newlines=True
+                    )
+                    commit_hash, _ = cmd.communicate()
+                    to_keep[tool].add(commit_hash.rstrip())
+
         removing_anything = False
-        for current, base in [(rust_current, "rust"), (cargo_current, "cargo")]:
-            base = path.join(self.context.sharedir, base)
+        for tool in ["rust", "cargo"]:
+            base = path.join(self.context.sharedir, tool)
             for name in os.listdir(base):
-                if name != current:
+                # We append `-alt` if LLVM assertions aren't enabled,
+                # so use just the commit hash itself.
+                # This may occasionally leave an extra nightly behind
+                # but won't remove too many nightlies.
+                if name.partition('-')[0] not in to_keep[tool]:
                     removing_anything = True
-                    name = path.join(base, name)
+                    full_path = path.join(base, name)
                     if force:
-                        print("Removing " + name)
-                        if os.path.isdir(name):
-                            shutil.rmtree(name)
-                        else:
-                            os.remove(name)
+                        print("Removing {}".format(full_path))
+                        delete(full_path)
                     else:
-                        print("Would remove " + name)
+                        print("Would remove {}".format(full_path))
         if not removing_anything:
             print("Nothing to remove.")
         elif not force:

python/servo/package_commands.py

@@ -33,13 +33,7 @@
     is_windows,
     get_browserhtml_path,
 )
-
-
-def delete(path):
-    try:
-        os.remove(path)         # Succeeds if path was a file
-    except OSError:             # Or, if path was a directory...
-        shutil.rmtree(path)     # Remove it and all its contents.
+from servo.util import delete
 
 
 def otool(s):

python/servo/util.py

@@ -10,16 +10,24 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
-import os.path as path
+import os.path
 import platform
-import sys
+import shutil
 from socket import error as socket_error
 import StringIO
+import sys
 import tarfile
 import zipfile
 import urllib2
 
 
+def delete(path):
+    if os.path.isdir(path) and not os.path.islink(path):
+        shutil.rmtree(path)
+    else:
+        os.remove(path)
+
+
 def host_platform():
     os_type = platform.system().lower()
     if os_type == "linux":
@@ -126,7 +134,7 @@ def download_bytes(desc, src):
 def download_file(desc, src, dst):
     tmp_path = dst + ".part"
     try:
-        start_byte = path.getsize(tmp_path)
+        start_byte = os.path.getsize(tmp_path)
         with open(tmp_path, 'ab') as fd:
             download(desc, src, fd, start_byte=start_byte)
     except os.error:
@@ -143,8 +151,8 @@ def extract(src, dst, movedir=None):
 
     if movedir:
         for f in os.listdir(movedir):
-            frm = path.join(movedir, f)
-            to = path.join(dst, f)
+            frm = os.path.join(movedir, f)
+            to = os.path.join(dst, f)
             os.rename(frm, to)
         os.rmdir(movedir)