Add: a way to clean up old nightlies (#31)

cdn/README.md

@@ -56,3 +56,9 @@ There are several keys per entry:
 - `sort`: optional (default: `version`).
   - `normal`: use "abc" sorting.
   - `version`: use version sorting: "1.9-alpha1" < "1.9-beta2" < "1.9-RC3" < "1.9" < "1.10".
+- `cleanup-after`: optional (default: `never`)
+  - `never`: never cleanup this folder.
+  - `Nmo` or `Nd`: after how many months/days a version can be removed.
+- `cleanup-keep`: optional (default: none)
+  - `source`: source files are always kept (binaries are removed).
+  - `N`: how many entries should be kept, even if expired.

cdn/cdn_generator/__main__.py

@@ -1,6 +1,7 @@
 import click
 import yaml
 
+from cdn_generator.cleanup import cleanup_category
 from cdn_generator.common_files import generate_common_files
 from cdn_generator.helpers import set_bucket_id
 from cdn_generator.latest_yaml import (
@@ -20,7 +21,9 @@
 @click.command(context_settings=CONTEXT_SETTINGS)
 @click.option("--bucket-id", help="Id of the S3 Bucket", required=True)
 @click.option("--new-release", help="Folder of the new release")
-def main(bucket_id, new_release):
+@click.option("--cleanup", help="Cleanup folders below --new-release folder", is_flag=True)
+@click.option("--cleanup-confirm", help="Confirm the generated list of files that would be deleted", is_flag=True)
+def main(bucket_id, new_release, cleanup, cleanup_confirm):
     set_bucket_id(bucket_id)
 
     with open("config.yaml", "r") as f:
@@ -49,6 +52,22 @@ def main(bucket_id, new_release):
         all_versions.extend(category_versions)
 
         if not skip_write:
+            if cleanup:
+                deleting = cleanup_category(
+                    category, category_config, filter_path=new_release, do_delete=cleanup_confirm
+                )
+                if not cleanup_confirm and deleting:
+                    with open("cleanup.txt", "w") as fp:
+                        for file in deleting:
+                            fp.write(f"{file}\n")
+
+                    print("In 'cleanup.txt' are all the files this script wants to delete.")
+                    print(
+                        "Please validate this list and rerun this script with '--cleanup-confirm' "
+                        "to actually delete those files."
+                    )
+                    return
+
             generate_manifests(category, category_config, filter_path=new_release)
             generate_directory_listing(category, config=category_config, filter_path=new_release)
 

cdn/cdn_generator/cleanup.py

@@ -0,0 +1,103 @@
+import datetime
+import dateutil.relativedelta
+import os
+
+from cdn_generator.helpers import delete_files, list_folder
+from cdn_generator.manifest_yaml import _generate_manifest
+from cdn_generator.directory_listing import _generate_directory_listing_single
+
+
+def _cleanup_directory(folder, keep_source, do_delete):
+    files, folders = list_folder(folder)
+    if len(folders) != 0:
+        raise Exception(f"Unexpected folders while cleaning up in {folder}")
+
+    deleting = []
+    for file in files:
+        if keep_source:
+            # Keep the source package.
+            if "-source.tar.xz" in file:
+                continue
+            # As this folder won't be empty, keep the files that indicate what this folder is about too.
+            if file in (
+                "changelog.txt",
+                "index.html",
+                "manifest.yaml",
+                "README.md",
+                "readme.txt",
+                "README.txt",
+                "released.txt",
+            ):
+                continue
+
+        fullname = f"{folder}/{file}"
+        deleting.append(fullname)
+
+    if not deleting:
+        return deleting
+
+    if do_delete:
+        print(f"Deleting {folder} ...")
+        delete_files(deleting)
+    else:
+        print(f"Collecting what to delete for {folder} ...")
+
+    return deleting
+
+
+def cleanup_category(category, config, filter_path=None, do_delete=False):
+    after = config["cleanup-after"]
+    if after.endswith("mo"):
+        after = dateutil.relativedelta.relativedelta(months=int(after[:-2]))
+    elif after.endswith("d"):
+        after = dateutil.relativedelta.relativedelta(days=int(after[:-1]))
+
+    cutoff_date = datetime.datetime.today() - after
+    cutoff_date = f"{cutoff_date.year}{cutoff_date.month:02d}{cutoff_date.day:02d}"
+
+    deleting = []
+
+    if config.get("subfolders") is not None:
+        _, folders = list_folder(category)
+        # Make sure we don't recurse a second time
+        old_subfolders = config["subfolders"]
+        config["subfolders"] = None
+
+        for folder in folders:
+            skip_write = filter_path and not filter_path.startswith(f"{category}/{folder}/")
+            if not skip_write:
+                deleting.extend(cleanup_category(f"{category}/{folder}", config, filter_path, do_delete))
+
+        config["subfolders"] = old_subfolders
+        return deleting
+
+    _, folders = list_folder(category)
+    folders = sorted(folders)
+
+    keep = config["cleanup-keep"]
+    keep_source = False
+    if keep == "source":
+        keep = 0
+        keep_source = True
+    else:
+        folders = folders[:-keep]
+
+    last_run_date = os.getenv("CLEANUP_LAST_RUN_DATE")
+
+    for folder in folders:
+        date = folder.split("-")[0]
+
+        # Skip anything before this date, as it is already considered done.
+        if last_run_date and date < last_run_date:
+            continue
+
+        if date < cutoff_date:
+            deleting_item = _cleanup_directory(f"{category}/{folder}", keep_source, do_delete)
+            deleting.extend(deleting_item)
+
+            if do_delete and deleting_item and keep_source:
+                files, _ = list_folder(f"{category}/{folder}")
+                _generate_manifest(category, folder, files, config)
+                _generate_directory_listing_single(f"{category}/{folder}", False)
+
+    return deleting

cdn/cdn_generator/helpers.py

@@ -65,6 +65,16 @@ def get_content(key):
     return content["Body"].read().decode().strip()
 
 
+def delete_files(files):
+    client.delete_objects(
+        Bucket=BUCKET,
+        Delete={
+            "Objects": [{"Key": file} for file in files],
+            "Quiet": True,
+        },
+    )
+
+
 def get_name(folder, config):
     override_name = config.get("override-name")
     if override_name is None:

cdn/config.yaml

@@ -14,11 +14,15 @@ folders:
   subfolders: per-year
   override-name: in-folder-name
   sort: normal
+  cleanup-after: 3mo
+  cleanup-keep: source
 
 - name: openttd-branches
   subfolders: per-name
   override-name: in-folder-name
   sort: normal
+  cleanup-after: 1mo
+  cleanup-keep: 1
 
 - name: grfcodec-nightlies
   override-name: nightly
@@ -27,11 +31,17 @@ folders:
 - name: opengfx-nightlies
   override-name: in-folder-name
   sort: normal
+  cleanup-after: 3mo
+  cleanup-keep: 3
 
 - name: opensfx-nightlies
   override-name: in-folder-name
   sort: normal
+  cleanup-after: 3mo
+  cleanup-keep: 3
 
 - name: openmsx-nightlies
   override-name: in-folder-name
   sort: normal
+  cleanup-after: 3mo
+  cleanup-keep: 3