metadata files routines

pyproject.toml

@@ -19,6 +19,8 @@ oras = "^0.2.38"
 pygit2 = "^1.18.2"
 pygments = "^2.19.2"
 PyYAML = "^6.0.2"
+gitpython = "^3.1.45"
+retrying = "^1.4.2"
 
 [tool.poetry.group.dev.dependencies]
 bandit = "^1.8.6"

src/gardenlinux/constants.py

@@ -155,3 +155,14 @@
 
 GLVD_BASE_URL = "https://glvd.ingress.glvd.gardnlinux.shoot.canary.k8s-hana.ondemand.com/v1"
 GL_DEB_REPO_BASE_URL = "https://packages.gardenlinux.io/gardenlinux"
+
+GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME = "gardenlinux-github-releases"
+
+# https://github.com/gardenlinux/gardenlinux/issues/3044
+# Empty string is the 'legacy' variant with traditional root fs and still needed/supported
+IMAGE_VARIANTS = ["", "_usi", "_tpm2_trustedboot"]
+
+# configuration for https://github.com/groodt/retrying
+RETRYING_MAX_ATTEMPTS = 5
+RETRYING_WAIT_EXPONENTIAL_MULTIPLIER = 1000
+RETRYING_WAIT_EXPONENTIAL_MAX = 16000

src/gardenlinux/github/release_notes/helpers.py

@@ -1,11 +1,25 @@
 import gzip
 import io
+import shutil
+from pathlib import Path
 
 import requests
+from git import Repo
 
 from gardenlinux.apt import DebsrcFile, GardenLinuxRepo
 from gardenlinux.apt.package_repo_info import compare_repo
-from gardenlinux.constants import GL_DEB_REPO_BASE_URL, REQUESTS_TIMEOUTS
+from gardenlinux.constants import (
+    GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME,
+    GL_DEB_REPO_BASE_URL,
+    IMAGE_VARIANTS,
+    REQUESTS_TIMEOUTS,
+)
+from gardenlinux.features import CName
+from gardenlinux.flavors import Parser as FlavorsParser
+from gardenlinux.logger import LoggerSetup
+from gardenlinux.s3 import S3Artifacts
+
+LOGGER = LoggerSetup.get_logger("gardenlinux.github.release_notes.helpers", "INFO")
 
 
 def get_package_list(gardenlinux_version):
@@ -33,3 +47,72 @@ def compare_apt_repo_versions(previous_version, current_version):
     for pkg in pkg_diffs:
         output += f"|{pkg[0]} | {pkg[1] if pkg[1] is not None else '-'} | {pkg[2] if pkg[2] is not None else '-'} |\n"
     return output
+
+
+def download_all_metadata_files(version, commitish):
+    repo = Repo(".")
+    commit = repo.commit(commitish)
+    flavors_data = commit.tree["flavors.yaml"].data_stream.read().decode("utf-8")
+    flavors = FlavorsParser(flavors_data).filter(only_publish=True)
+
+    local_dest_path = Path("s3_downloads")
+    if local_dest_path.exists():
+        shutil.rmtree(local_dest_path)
+    local_dest_path.mkdir(mode=0o755, exist_ok=False)
+
+    s3_artifacts = S3Artifacts(GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME)
+
+    commitish_short = commitish[:8]
+
+    for flavor in flavors:
+        cname = CName(flavor[1], flavor[0], "{0}-{1}".format(version, commitish_short))
+        LOGGER.debug(f"{flavor=} {version=} {commitish=}")
+        # Filter by image variants - only download if the flavor matches one of the variants
+        flavor_matches_variant = False
+        for variant_suffix in IMAGE_VARIANTS:
+            if variant_suffix == "":
+                last_part = cname.cname.split("-")[-1]
+                if "_" not in last_part:
+                    flavor_matches_variant = True
+                    break
+            elif variant_suffix in cname.cname:
+                # Specific variant (any non-empty string in IMAGE_VARIANTS)
+                flavor_matches_variant = True
+                break
+
+        if not flavor_matches_variant:
+            LOGGER.info(
+                f"Skipping flavor {cname.cname} - not matching image variants filter"
+            )
+            continue
+
+        try:
+            download_metadata_file(
+                s3_artifacts, cname.cname, version, commitish_short, local_dest_path
+            )
+        except IndexError:
+            LOGGER.warn(f"No artifacts found for flavor {cname.cname}, skipping...")
+            continue
+
+    return [str(artifact) for artifact in local_dest_path.iterdir()]
+
+
+def download_metadata_file(
+    s3_artifacts, cname, version, commitish_short, artifacts_dir
+):
+    """
+    Download metadata file (s3_metadata.yaml)
+    """
+    LOGGER.debug(
+        f"{s3_artifacts=} | {cname=} | {version=} | {commitish_short=} | {artifacts_dir=}"
+    )
+    release_object = list(
+        s3_artifacts._bucket.objects.filter(
+            Prefix=f"meta/singles/{cname}-{version}-{commitish_short}"
+        )
+    )[0]
+    LOGGER.debug(f"{release_object.bucket_name=} | {release_object.key=}")
+
+    s3_artifacts.bucket.download_file(
+        release_object.key, artifacts_dir.joinpath(f"{cname}.s3_metadata.yaml")
+    )

src/gardenlinux/s3/bucket.py

@@ -12,7 +12,9 @@
 from typing import Any, Optional
 
 import boto3
+from retrying import retry
 
+from ..constants import RETRYING_MAX_ATTEMPTS, RETRYING_WAIT_EXPONENTIAL_MAX, RETRYING_WAIT_EXPONENTIAL_MULTIPLIER
 from ..logger import LoggerSetup
 
 
@@ -88,6 +90,9 @@ class tree for self).
 
         return getattr(self._bucket, name)
 
+    @retry(stop_max_attempt_number=RETRYING_MAX_ATTEMPTS,
+           wait_exponential_multiplier=RETRYING_WAIT_EXPONENTIAL_MULTIPLIER,
+           wait_exponential_max=RETRYING_WAIT_EXPONENTIAL_MAX)
     def download_file(self, key, file_name, *args, **kwargs):
         """
         boto3: Download an S3 object to a file.