setup.py, CI: avoid ENOSPC on Github Actions

The Github Actions disk space can get filled up if we test all kernel
flavors. So, delete kernels after testing when running in Github
Actions. However, if the download speed is far greater than the speed of
tests, then there is also the possibility that the disk space will be
filled up by the download thread, before the tests will run and delete
the older kernels. So we also need to ensure the download thread can't
get too far ahead of the tests, which we implement by setting a max size
on the download queue.

Signed-off-by: Stephen Brennan <stephen.s.brennan@oracle.com>

setup.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 import re
 import shlex
+import shutil
 import subprocess
 import sys
 import sysconfig
@@ -263,7 +264,9 @@ def run(self):
         from vmtest.config import ARCHITECTURES, Kernel, local_kernel
         from vmtest.download import DownloadCompiler, DownloadKernel, download_in_thread
 
-        if os.getenv("GITHUB_ACTIONS") == "true":
+        in_github_actions = os.getenv("GITHUB_ACTIONS") == "true"
+
+        if in_github_actions:
 
             @contextlib.contextmanager
             def github_workflow_group(title):
@@ -292,7 +295,16 @@ def github_workflow_group(title):
                         to_download.append(
                             DownloadKernel(ARCHITECTURES["x86_64"], pattern)
                         )
-            with download_in_thread(Path(self.vmtest_dir), to_download) as downloads:
+
+            # Downloading too many files before they can be used for testing runs the
+            # risk of filling up the limited disk space is Github Actions. Set a limit
+            # of no more than 5 files which can be downloaded ahead of time. This is a
+            # magic number which is inexact, but works well enough.
+            max_pending_kernels = 5 if in_github_actions else 0
+
+            with download_in_thread(
+                Path(self.vmtest_dir), to_download, max_pending_kernels
+            ) as downloads:
                 downloads_it = iter(downloads)
 
                 if to_download:
@@ -334,6 +346,12 @@ def github_workflow_group(title):
                         logger.info("Passed: %s", ", ".join(passed))
                     if failed:
                         logger.error("Failed: %s", ", ".join(failed))
+
+                    # Github Actions has limited disk space. Once tested, we
+                    # will not use the kernel again, so delete it.
+                    if in_github_actions:
+                        logger.info("Deleting kernel %s", kernel.release)
+                        shutil.rmtree(kernel.path)
         except urllib.error.HTTPError as e:
             if e.code == 403:
                 print(e, file=sys.stderr)

vmtest/__main__.py

@@ -5,6 +5,7 @@
 import os
 from pathlib import Path
 import shlex
+import shutil
 import subprocess
 import sys
 from typing import Dict, List, TextIO
@@ -243,7 +244,19 @@ def add_kernel(arch: Architecture, pattern: str) -> None:
 
     progress = _ProgressPrinter(sys.stderr)
 
-    with download_in_thread(args.directory, to_download) as downloads:
+    in_github_actions = os.getenv("GITHUB_ACTIONS") == "true"
+
+    # Downloading too many files before they can be used for testing runs the
+    # risk of filling up the limited disk space is Github Actions. Set a limit
+    # of no more than 5 files which can be downloaded ahead of time. This is a
+    # magic number which is inexact, but works well enough.
+    # Note that Github Actions does not run vmtest via this script currently,
+    # but may in the future.
+    max_pending_kernels = 5 if in_github_actions else 0
+
+    with download_in_thread(
+        args.directory, to_download, max_pending_kernels
+    ) as downloads:
         for arch in architectures:
             if arch is HOST_ARCHITECTURE:
                 subprocess.check_call(
@@ -337,4 +350,7 @@ def add_kernel(arch: Architecture, pattern: str) -> None:
             except LostVMError as e:
                 print("error:", e, file=sys.stderr)
                 status = -1
+
+            if in_github_actions:
+                shutil.rmtree(kernel.path)
             progress.update(kernel.arch.name, kernel.release, status == 0)

vmtest/download.py

@@ -284,9 +284,11 @@ def _download_thread(
 
 @contextmanager
 def download_in_thread(
-    download_dir: Path, downloads: Iterable[Download]
+    download_dir: Path, downloads: Iterable[Download], max_pending_kernels: int = 0
 ) -> Generator[Iterator[Downloaded], None, None]:
-    q: "queue.Queue[Union[Downloaded, Exception]]" = queue.Queue()
+    q: "queue.Queue[Union[Downloaded, Exception]]" = queue.Queue(
+        maxsize=max_pending_kernels
+    )
 
     def aux() -> Iterator[Downloaded]:
         while True: