Merge 19fcdb6 into 00a6646

.github/workflows/build.yml

@@ -370,6 +370,10 @@ jobs:
       - name: "[macOS] Sign, Notarize, and Staple Package"
         if: runner.os == 'macOS' && env.MACOS_CODESIGN_OPENSSL_PASSWORD != null && env.MACOS_CODESIGN_CERTIFICATE_PASSWORD != null && env.APPLE_APP_SPECIFIC_PASSWORD != null
         run: packaging/macos/sign_notarize_staple.sh build/*.dmg
+        # TODO: This timeout may be a bit too conservative under some
+        # circumstances. Feel free to increase the timeout once the
+        # notarization step is fixed.
+        timeout-minutes: 5
         env:
           APPLE_ID_USERNAME: rryan@mixxx.org
           APPLE_BUNDLE_ID: org.mixxx.mixxx
@@ -477,6 +481,9 @@ jobs:
     name: "Update manifest file on download server"
     runs-on: ubuntu-latest
     needs: build
+    # Always run this job, even if one or more jobs from the `build` jobs
+    # fail to allow partial updates of the manifest.
+    if: always()
     steps:
       - name: "Check out repository"
         uses: actions/checkout@v4
@@ -501,6 +508,7 @@ jobs:
             export DEPLOY_PATH='snapshots/{git_branch}/manifest.json';
           fi;
           python3 tools/deploy.py generate-manifest
+          --update
           --output-dir 'deploy/'
           --dest-path "${DEPLOY_PATH}"
           --dest-url 'https://downloads.mixxx.org'

tools/deploy.py

@@ -4,6 +4,7 @@
 import functools
 import hashlib
 import json
+import logging
 import os
 import pathlib
 import shutil
@@ -40,12 +41,10 @@ def url_exists(url):
 
 def url_download_json(url):
     """Returns the JSON object from the given URL or return None."""
-    try:
-        resp = url_fetch(url)
-        manifest_data = resp.read().decode()
-    except IOError:
-        return None
-
+    resp = url_fetch(url)
+    if resp.status != 200:
+        raise IOError(f"Server responded with HTTP status {resp.status}")
+    manifest_data = resp.read().decode().strip()
     return json.loads(manifest_data)
 
 
@@ -116,6 +115,8 @@ def slug(text):
 
 
 def prepare_deployment(args):
+    logger = logging.getLogger(__name__)
+
     # Get artifact and build metadata
     file_stat = os.stat(args.file)
     file_sha256 = sha256(args.file)
@@ -157,7 +158,7 @@ def prepare_deployment(args):
     download_slug, package_slug = args.slug
 
     # Build destination path scheme
-    print(f"Destination path pattern: {args.dest_path}")
+    logger.info("Destination path pattern: %s", args.dest_path)
     destpath = args.dest_path.format(
         filename=os.path.basename(args.file),
         filename_without_ext=filename_without_ext,
@@ -168,7 +169,7 @@ def prepare_deployment(args):
         package_slug=package_slug,
         download_slug=download_slug,
     )
-    print(f"Destination path: {destpath}")
+    logger.info("Destination path: %s", destpath)
 
     # Move files to deploy in place and create sha256sum file
     output_destpath = os.path.join(args.output_dir, destpath)
@@ -187,10 +188,8 @@ def prepare_deployment(args):
     )
 
     # Show metadata and files to deploy
-    print("Metadata: ", json.dumps(metadata, indent=2, sort_keys=True))
-    print("Files:")
-    for path in tree(args.output_dir):
-        print(path)
+    logger.info("Metadata: %s", json.dumps(metadata, indent=2, sort_keys=True))
+    logger.info("Files: \n%s", "\n".join(tree(args.output_dir)))
 
     # Write metadata to GitHub Actions step output, so that it can be used for
     # manifest creation in the final job after all builds finished.
@@ -209,16 +208,16 @@ def prepare_deployment(args):
 
 def collect_manifest_data(job_data):
     """Parse the job metadata dict and return the manifest data."""
+    logger = logging.getLogger(__name__)
     job_result = job_data["result"]
-    print(f"Build job result: {job_result}")
-    assert job_result == "success"
+    logger.info("Build job result: %s", job_result)
 
     manifest_data = {}
     for output_name, output_data in job_data["outputs"].items():
         # Filter out unrelated job outputs that don't start with "artifact-".
         prefix, _, artifact_slug = output_name.partition("-")
         if prefix != "artifact" or not artifact_slug:
-            print(f"Ignoring output '{output_name}'...")
+            logger.info("Ignoring output '%s'...", output_name)
             continue
         artifact_data = json.loads(output_data)
 
@@ -235,6 +234,7 @@ def collect_manifest_data(job_data):
 
 
 def generate_manifest(args):
+    logger = logging.getLogger(__name__)
     try:
         commit_id = os.getenv("GITHUB_SHA")
     except KeyError:
@@ -247,9 +247,9 @@ def generate_manifest(args):
     }
 
     # Build destination path scheme
-    print(f"Destination path pattern: {args.dest_path}")
+    logger.info("Destination path pattern: %s", args.dest_path)
     destpath = args.dest_path.format_map(format_data)
-    print(f"Destination path: {destpath}")
+    logger.info("Destination path: %s", destpath)
 
     # Create the deployment directory
     output_destpath = os.path.join(args.output_dir, destpath)
@@ -258,37 +258,45 @@ def generate_manifest(args):
     # Parse the JOB_DATA JSON data, generate the manifest data and print it
     job_data = json.loads(os.environ["JOB_DATA"])
     manifest_data = collect_manifest_data(job_data)
-    print("Manifest:", json.dumps(manifest_data, indent=2, sort_keys=True))
-
-    # Write the manifest.json for subsequent deployment to the server
-    with open(output_destpath, mode="w") as fp:
-        json.dump(manifest_data, fp, indent=2, sort_keys=True)
 
     # If possible, check if the remote manifest is the same as our local one
-    remote_manifest_data = None
+    remote_manifest_data = {}
     if args.dest_url:
         # Check if generated manifest.json file differs from the one that
         # is currently deployed.
         manifest_url = f"{args.dest_url}/{destpath}"
         manifest_url = manifest_url.format_map(format_data)
 
         try:
-            remote_manifest_data = url_fetch(manifest_url)
+            remote_manifest_data = url_download_json(manifest_url) or {}
         except IOError:
-            pass
+            logger.error("Fetching remote manifest failed!")
+
+    if args.update:
+        for key, value in remote_manifest_data.items():
+            if key not in manifest_data:
+                manifest_data[key] = value
+
+    logger.info(
+        "Manifest: %s", json.dumps(manifest_data, indent=2, sort_keys=True)
+    )
+
+    # Write the manifest.json for subsequent deployment to the server
+    with open(output_destpath, mode="w") as fp:
+        json.dump(manifest_data, fp, indent=2, sort_keys=True)
 
     # Skip deployment if the remote manifest is the same as the local one.
     if manifest_data != remote_manifest_data:
-        print("Remote manifest differs from local version.")
+        logger.info("Remote manifest differs from local version.")
         if os.getenv("CI") == "true":
             with open(os.environ["GITHUB_ENV"], mode="a") as fp:
                 fp.write("MANIFEST_DIRTY=1\n")
     else:
-        print("Remote manifest is the same as local version.")
+        logger.info(
+            "Remote manifest is the same as local version, skipping upload."
+        )
 
-    print("Files:")
-    for path in tree(args.output_dir):
-        print(path)
+    logger.info("Files:\n%s", "\n".join(tree(args.output_dir)))
 
     return 0
 
@@ -350,9 +358,25 @@ def main(argv=None):
     manifest_parser.add_argument(
         "--dest-url", action="store", help="Destination URL prefix"
     )
+    manifest_parser.add_argument(
+        "--update",
+        action="store_true",
+        help="Fetch the remote manifest and update it ",
+    )
+    manifest_parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_const",
+        dest="loglevel",
+        const=logging.DEBUG,
+        default=logging.INFO,
+        help="Fetch the remote manifest and update it ",
+    )
 
     args = parser.parse_args(argv)
 
+    logging.basicConfig(level=args.loglevel, format="%(message)s")
+
     if os.path.exists(args.output_dir):
         if not os.path.isdir(args.output_dir):
             raise OSError("Output dir is not a directory!")