Copy all packages and catch those that couldn't be copied (infra)

tools/release/lp_copy_packages.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # This file is part of Checkbox.
 #
-# Copyright 2023 Canonical Ltd.
+# Copyright 2023-2025 Canonical Ltd.
 # Written by:
 #   Sylvain Pineau <sylvain.pineau@canonical.com>
 #   Massimiliano Girardi <massimiliano.girardi@canonical.com>
@@ -26,9 +26,8 @@
 Note: This script uses the LP_CREDENTIALS environment variable
 """
 import sys
-import datetime
+import lazr
 import argparse
-import itertools
 
 from utils import get_launchpad_client
 
@@ -48,65 +47,52 @@
     return ppa_owner.getPPAByName(name=ppa_name)
 
 
-def get_checkbox_packages(ppa):
+def copy_packages(source_owner, source_ppa, dest_owner, dest_ppa):
     """
-    Get all the most recent checkbox packages on the PPA that are still current
-
-    A source package is still current when it has not been superseeded by
-    another. The filtering here is done to avoid copying over outdated
-    packages to the target PPA
-    """
-    # Note: this is not the same as ppa.getPublishedSources(status="Published")
-    #       the reason is that if a package is Published but for a not
-    #       supported distribution, say Lunar, copying it over will trigger an
-    #       error. When a distribution support is dropped, Launchpad will
-    #       automatically stop building for it and start a grace period for
-    #       updates. This ensures there will always be a pocket of Superseeded
-    #       packages between Published packages for unsupported distro and
-    #       current ones
-    all_published_sources = ppa.getPublishedSources(
-        source_name="checkbox", order_by_date=True
-    )
-    # this filters out superseeded packages AND Published packages that are no
-    # longer current (as they are not being built anymore by Launchpad)
-    return itertools.takewhile(
-        lambda x: x.date_superseded is None, all_published_sources
-    )
-
-
-def copy_checkbox_packages(source_owner, source_ppa, dest_owner, dest_ppa):
-    """
-    Copy Checkbox packages from a source PPA to a destination PPA without
+    Copy all packages from a source PPA to a destination PPA without
     rebuilding.
     """
     lp = get_launchpad_client()
 
     source_ppa = get_ppa(lp, source_ppa, source_owner)
     dest_ppa = get_ppa(lp, dest_ppa, dest_owner)
 
-    packages = get_checkbox_packages(source_ppa)
+    packages = source_ppa.getPublishedSources(
+        order_by_date=True, status="Published"
+    )
 
     # Copy each package from the source PPA to the destination PPA,
     # without rebuilding them
     for package in packages:
-        dest_ppa.copyPackage(
-            from_archive=source_ppa,
-            include_binaries=True,
-            to_pocket=package.pocket,
-            source_name=package.source_package_name,
-            version=package.source_package_version,
-        )
-        print(
-            f"Copied {package.source_package_name} "
-            f"version {package.source_package_version} "
-            f"from {source_ppa} to {dest_ppa} "
-            "(without rebuilding)"
-        )
+        try:
+            dest_ppa.copyPackage(
+                from_archive=source_ppa,
+                include_binaries=True,
+                to_pocket=package.pocket,
+                source_name=package.source_package_name,
+                version=package.source_package_version,
+            )
+            print(
+                f"Copied {package.source_package_name} "
+                f"version {package.source_package_version} "
+                f"from {source_ppa} to {dest_ppa} "
+                "(without rebuilding)"
+            )
+        except lazr.restfulclient.errors.BadRequest as e:
+            # This is expected when trying to copy a package to a target distro
+            # that is EOL and can be safely ignored
+            if "is obsolete and will not accept new uploads" not in str(e):
+                raise
+            print(
+                f"Skipped {package.source_package_name} "
+                f"version {package.source_package_version} "
+                "(target series is obsolete)"
+            )
 
 
 def main(argv):
     args = parse_args(argv)
-    copy_checkbox_packages(
+    copy_packages(
         args.source_owner,
         args.source_ppa,
         args.dest_owner,

tools/release/lp_update_recipe.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # This file is part of Checkbox.
 #
-# Copyright 2023 Canonical Ltd.
+# Copyright 2023-2025 Canonical Ltd.
 # Written by:
 #   Sylvain Pineau <sylvain.pineau@canonical.com>
 #   Massimiliano Girardi <massimiliano.girardi@canonical.com>
@@ -29,7 +29,44 @@
 import textwrap
 import argparse
 
-from utils import get_source_build_recipe
+from utils import get_source_build_recipe, get_launchpad_client
+
+
+def get_active_series():
+    # this is hardcoded because supporting a new release requires a bit of manual
+    # effort (copying support packages to the new series and doing the first build
+    # dance of building packages in dependency order)
+    desired_series = {
+        "bionic",
+        "focal",
+        "jammy",
+        "noble",
+        "plucky",
+        "questing",
+    }
+    lp = get_launchpad_client()
+    active_series = {
+        series.name for series in lp.projects["ubuntu"].series if series.active
+    }
+
+    outdated_series = desired_series - active_series
+    if outdated_series:
+        print(
+            "The following series were desired but are now outdated:\n- ",
+            end="",
+        )
+        print("\n- ".join(outdated_series))
+
+    undesired_active = active_series - desired_series
+    if undesired_active:
+        print(
+            "The following series weren't desired but are active:\n- ", end=""
+        )
+        print("\n- ".join(undesired_active))
+
+    possible_series = desired_series & active_series
+    api_formatter = "https://api.launchpad.net/devel/ubuntu/{}"
+    return [api_formatter.format(x) for x in possible_series]
 
 
 def get_build_path(recipe_name: str) -> str:
@@ -70,6 +107,10 @@ def update_build_recipe(
     lp_recipe = get_source_build_recipe(project_name, recipe_name)
     new_recipe = get_updated_build_recipe(recipe_name, version, revision)
     lp_recipe.recipe_text = new_recipe
+    # this is important because while debugging, one may inadvertedly leave a
+    # not-yet-working daily build enabled, this makes the daily builds fail for
+    # no reason
+    lp_recipe.distroseries = get_active_series()
     lp_recipe.lp_save()
     print(f"Updated build recipe: {lp_recipe.web_link}")
     print(new_recipe)

tools/release/test_lp_copy_packages.py

@@ -1,4 +1,6 @@
 import unittest
+import lazr
+
 from unittest.mock import patch, MagicMock
 
 import lp_copy_packages
@@ -12,18 +14,35 @@ def test_main(self, get_launchpad_client_mock):
         checkbox_dev_user = MagicMock()
         lp_client.people = {"checkbox-dev": checkbox_dev_user}
 
-        source_to_copy = MagicMock(date_superseded=None)
-        source_no_copy_superseeded = MagicMock(date_superseded="some date")
-        source_no_copy_outdated_distro = MagicMock(date_superseded=None)
+        source_to_copy = MagicMock(
+            date_superseded=None, source_package_name="up to date package"
+        )
+        source_no_copy_outdated_distro = MagicMock(
+            date_superseded=None, source_package_name="outdated source"
+        )
 
         ppas = checkbox_dev_user.getPPAByName()
         ppas.getPublishedSources.return_value = [source_to_copy] * 5 + [
-            source_no_copy_superseeded,
             source_no_copy_outdated_distro,
         ]
-
+        copied = 0
+
+        def fail_on_outdated(**kwargs):
+            nonlocal copied
+            source_name = kwargs["source_name"]
+            print(source_name)
+            if source_name == "outdated source":
+                raise lazr.restfulclient.errors.BadRequest(
+                    response=MagicMock(
+                        items=lambda: [], status=400, reason="some"
+                    ),
+                    content=b"distro is obsolete and will not accept new uploads",
+                )
+            copied += 1
+
+        ppas.copyPackage = fail_on_outdated
         lp_copy_packages.main(
             ["checkbox-dev", "beta", "checkbox-dev", "stable"]
         )
 
-        self.assertEqual(ppas.copyPackage.call_count, 5)
+        self.assertEqual(copied, 5)

tools/release/test_lp_update_recipe.py

@@ -38,6 +38,10 @@ def test_get_updated_build_recipe(self):
 
     @patch("lp_update_recipe.get_source_build_recipe")
     @patch("lp_update_recipe.print")
+    @patch(
+        "lp_update_recipe.get_active_series",
+        new=MagicMock(return_value="noble"),
+    )
     def test_main(self, print_mock, get_build_path_mock):
         lp_recipe = get_build_path_mock()
 
@@ -56,3 +60,26 @@ def test_main(self, print_mock, get_build_path_mock):
         self.assertIn("checkbox-support", lp_recipe.recipe_text)
         self.assertIn("abcd", lp_recipe.recipe_text)
         self.assertTrue(lp_recipe.lp_save.called)
+
+    @patch("lp_update_recipe.print")
+    @patch("lp_update_recipe.get_launchpad_client")
+    def test_get_active_series(self, get_launchpad_client_mock, print_mock):
+        lp_mock = get_launchpad_client_mock()
+        # if this broke, 10+yrs went by and this is still in use. Good :)
+        # Update the following line with the latest LTS name
+        active_release_mock = MagicMock(active=True)
+        active_release_mock.name = "noble"
+        active_undesired_mock = MagicMock(active=True)
+        active_undesired_mock.name = " <this release is undesired> "
+        lp_mock.projects["ubuntu"].series = [
+            active_release_mock,
+            active_undesired_mock,
+        ]
+        returned = lp_update_recipe.get_active_series()
+        self.assertTrue(any(active_release_mock.name in x for x in returned))
+        log = "\n".join(str(x) for x in print_mock.call_args_list)
+        # active but undesired was reported
+        self.assertIn(active_undesired_mock.name, log)
+        # this assumes more than 1 distro is active, if this fails you are
+        # desiring only 1 distro which is the latest lts from before
+        self.assertIn("outdated", log)