From b23882c6ee5ba5c53d7ec40d892e3749c5990987 Mon Sep 17 00:00:00 2001 From: Hook25 Date: Mon, 14 Jul 2025 12:18:22 +0200 Subject: [PATCH 1/3] Copy all packages and catch those that couldn't be copied --- tools/release/lp_copy_packages.py | 76 ++++++++++---------------- tools/release/lp_update_recipe.py | 25 +++++++-- tools/release/test_lp_copy_packages.py | 35 ++++++++---- 3 files changed, 74 insertions(+), 62 deletions(-) diff --git a/tools/release/lp_copy_packages.py b/tools/release/lp_copy_packages.py index 2d1761bbff..5e933de482 100755 --- a/tools/release/lp_copy_packages.py +++ b/tools/release/lp_copy_packages.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # This file is part of Checkbox. # -# Copyright 2023 Canonical Ltd. +# Copyright 2023-2025 Canonical Ltd. # Written by: # Sylvain Pineau # Massimiliano Girardi @@ -26,9 +26,8 @@ Note: This script uses the LP_CREDENTIALS environment variable """ import sys -import datetime +import lazr import argparse -import itertools from utils import get_launchpad_client @@ -48,35 +47,9 @@ def get_ppa(lp, ppa_name: str, ppa_owner: str): return ppa_owner.getPPAByName(name=ppa_name) -def get_checkbox_packages(ppa): +def copy_packages(source_owner, source_ppa, dest_owner, dest_ppa): """ - Get all the most recent checkbox packages on the PPA that are still current - - A source package is still current when it has not been superseeded by - another. The filtering here is done to avoid copying over outdated - packages to the target PPA - """ - # Note: this is not the same as ppa.getPublishedSources(status="Published") - # the reason is that if a package is Published but for a not - # supported distribution, say Lunar, copying it over will trigger an - # error. When a distribution support is dropped, Launchpad will - # automatically stop building for it and start a grace period for - # updates. This ensures there will always be a pocket of Superseeded - # packages between Published packages for unsupported distro and - # current ones - all_published_sources = ppa.getPublishedSources( - source_name="checkbox", order_by_date=True - ) - # this filters out superseeded packages AND Published packages that are no - # longer current (as they are not being built anymore by Launchpad) - return itertools.takewhile( - lambda x: x.date_superseded is None, all_published_sources - ) - - -def copy_checkbox_packages(source_owner, source_ppa, dest_owner, dest_ppa): - """ - Copy Checkbox packages from a source PPA to a destination PPA without + Copy all packages from a source PPA to a destination PPA without rebuilding. """ lp = get_launchpad_client() @@ -84,29 +57,40 @@ def copy_checkbox_packages(source_owner, source_ppa, dest_owner, dest_ppa): source_ppa = get_ppa(lp, source_ppa, source_owner) dest_ppa = get_ppa(lp, dest_ppa, dest_owner) - packages = get_checkbox_packages(source_ppa) + packages = source_ppa.getPublishedSources(order_by_date=True, status="Published") # Copy each package from the source PPA to the destination PPA, # without rebuilding them for package in packages: - dest_ppa.copyPackage( - from_archive=source_ppa, - include_binaries=True, - to_pocket=package.pocket, - source_name=package.source_package_name, - version=package.source_package_version, - ) - print( - f"Copied {package.source_package_name} " - f"version {package.source_package_version} " - f"from {source_ppa} to {dest_ppa} " - "(without rebuilding)" - ) + try: + dest_ppa.copyPackage( + from_archive=source_ppa, + include_binaries=True, + to_pocket=package.pocket, + source_name=package.source_package_name, + version=package.source_package_version, + ) + print( + f"Copied {package.source_package_name} " + f"version {package.source_package_version} " + f"from {source_ppa} to {dest_ppa} " + "(without rebuilding)" + ) + except lazr.restfulclient.errors.BadRequest as e: + # This is expected when trying to copy a package to a target distro + # that is EOL and can be safely ignored + if "is obsolete and will not accept new uploads" not in str(e): + raise + print( + f"Skipped {package.source_package_name} " + f"version {package.source_package_version} " + "(target series is obsolete)" + ) def main(argv): args = parse_args(argv) - copy_checkbox_packages( + copy_packages( args.source_owner, args.source_ppa, args.dest_owner, diff --git a/tools/release/lp_update_recipe.py b/tools/release/lp_update_recipe.py index 011abcea17..f76a3c2ab2 100755 --- a/tools/release/lp_update_recipe.py +++ b/tools/release/lp_update_recipe.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # This file is part of Checkbox. # -# Copyright 2023 Canonical Ltd. +# Copyright 2023-2025 Canonical Ltd. # Written by: # Sylvain Pineau # Massimiliano Girardi @@ -31,6 +31,18 @@ from utils import get_source_build_recipe +# this is hardcoded because supporting a new release requires a bit of manual +# effort (copying support packages to the new series and doing the first build +# dance of building packages in dependency order) +active_series = [ + "https://api.launchpad.net/devel/ubuntu/bionic", + "https://api.launchpad.net/devel/ubuntu/focal", + "https://api.launchpad.net/devel/ubuntu/jammy", + "https://api.launchpad.net/devel/ubuntu/noble", + "https://api.launchpad.net/devel/ubuntu/plucky", + "https://api.launchpad.net/devel/ubuntu/questing", +] + def get_build_path(recipe_name: str) -> str: # recipe name is in the form checkbox-{package}-{risk} @@ -70,6 +82,10 @@ def update_build_recipe( lp_recipe = get_source_build_recipe(project_name, recipe_name) new_recipe = get_updated_build_recipe(recipe_name, version, revision) lp_recipe.recipe_text = new_recipe + # this is important because while debugging, one may inadvertedly leave a + # not-yet-working daily build enabled, this makes the daily builds fail for + # no reason + lp_recipe.distroseries = active_series lp_recipe.lp_save() print(f"Updated build recipe: {lp_recipe.web_link}") print(new_recipe) @@ -84,8 +100,7 @@ def parse_args(argv: list[str]) -> argparse.Namespace: parser.add_argument( "--new-version", "-n", - help="New version to use in the recipe " - "(for debian changelog) and bzr tags.", + help="New version to use in the recipe " "(for debian changelog) and bzr tags.", required=True, ) parser.add_argument("--revision", help="Revision to build", required=True) @@ -94,9 +109,7 @@ def parse_args(argv: list[str]) -> argparse.Namespace: def main(argv): args = parse_args(argv) - update_build_recipe( - args.project, args.recipe, args.new_version, args.revision - ) + update_build_recipe(args.project, args.recipe, args.new_version, args.revision) if __name__ == "__main__": diff --git a/tools/release/test_lp_copy_packages.py b/tools/release/test_lp_copy_packages.py index 21545a4bae..da8c5bf988 100644 --- a/tools/release/test_lp_copy_packages.py +++ b/tools/release/test_lp_copy_packages.py @@ -1,4 +1,6 @@ import unittest +import lazr + from unittest.mock import patch, MagicMock import lp_copy_packages @@ -12,18 +14,31 @@ def test_main(self, get_launchpad_client_mock): checkbox_dev_user = MagicMock() lp_client.people = {"checkbox-dev": checkbox_dev_user} - source_to_copy = MagicMock(date_superseded=None) - source_no_copy_superseeded = MagicMock(date_superseded="some date") - source_no_copy_outdated_distro = MagicMock(date_superseded=None) + source_to_copy = MagicMock( + date_superseded=None, source_package_name="up to date package" + ) + source_no_copy_outdated_distro = MagicMock( + date_superseded=None, source_package_name="outdated source" + ) ppas = checkbox_dev_user.getPPAByName() ppas.getPublishedSources.return_value = [source_to_copy] * 5 + [ - source_no_copy_superseeded, source_no_copy_outdated_distro, ] - - lp_copy_packages.main( - ["checkbox-dev", "beta", "checkbox-dev", "stable"] - ) - - self.assertEqual(ppas.copyPackage.call_count, 5) + copied = 0 + + def fail_on_outdated(**kwargs): + nonlocal copied + source_name = kwargs["source_name"] + print(source_name) + if source_name == "outdated source": + raise lazr.restfulclient.errors.BadRequest( + response=MagicMock(items=lambda: [], status=400, reason="some"), + content=b"distro is obsolete and will not accept new uploads", + ) + copied += 1 + + ppas.copyPackage = fail_on_outdated + lp_copy_packages.main(["checkbox-dev", "beta", "checkbox-dev", "stable"]) + + self.assertEqual(copied, 5) From a466714553971229a2d7105912400a5bdac08b7c Mon Sep 17 00:00:00 2001 From: Hook25 Date: Mon, 14 Jul 2025 12:29:06 +0200 Subject: [PATCH 2/3] Something is wrong in my black sigh --- tools/release/lp_copy_packages.py | 4 +++- tools/release/lp_update_recipe.py | 7 +++++-- tools/release/test_lp_copy_packages.py | 8 ++++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/tools/release/lp_copy_packages.py b/tools/release/lp_copy_packages.py index 5e933de482..4d8ae1d006 100755 --- a/tools/release/lp_copy_packages.py +++ b/tools/release/lp_copy_packages.py @@ -57,7 +57,9 @@ def copy_packages(source_owner, source_ppa, dest_owner, dest_ppa): source_ppa = get_ppa(lp, source_ppa, source_owner) dest_ppa = get_ppa(lp, dest_ppa, dest_owner) - packages = source_ppa.getPublishedSources(order_by_date=True, status="Published") + packages = source_ppa.getPublishedSources( + order_by_date=True, status="Published" + ) # Copy each package from the source PPA to the destination PPA, # without rebuilding them diff --git a/tools/release/lp_update_recipe.py b/tools/release/lp_update_recipe.py index f76a3c2ab2..472c5f9d8f 100755 --- a/tools/release/lp_update_recipe.py +++ b/tools/release/lp_update_recipe.py @@ -100,7 +100,8 @@ def parse_args(argv: list[str]) -> argparse.Namespace: parser.add_argument( "--new-version", "-n", - help="New version to use in the recipe " "(for debian changelog) and bzr tags.", + help="New version to use in the recipe " + "(for debian changelog) and bzr tags.", required=True, ) parser.add_argument("--revision", help="Revision to build", required=True) @@ -109,7 +110,9 @@ def parse_args(argv: list[str]) -> argparse.Namespace: def main(argv): args = parse_args(argv) - update_build_recipe(args.project, args.recipe, args.new_version, args.revision) + update_build_recipe( + args.project, args.recipe, args.new_version, args.revision + ) if __name__ == "__main__": diff --git a/tools/release/test_lp_copy_packages.py b/tools/release/test_lp_copy_packages.py index da8c5bf988..f3752cd22f 100644 --- a/tools/release/test_lp_copy_packages.py +++ b/tools/release/test_lp_copy_packages.py @@ -33,12 +33,16 @@ def fail_on_outdated(**kwargs): print(source_name) if source_name == "outdated source": raise lazr.restfulclient.errors.BadRequest( - response=MagicMock(items=lambda: [], status=400, reason="some"), + response=MagicMock( + items=lambda: [], status=400, reason="some" + ), content=b"distro is obsolete and will not accept new uploads", ) copied += 1 ppas.copyPackage = fail_on_outdated - lp_copy_packages.main(["checkbox-dev", "beta", "checkbox-dev", "stable"]) + lp_copy_packages.main( + ["checkbox-dev", "beta", "checkbox-dev", "stable"] + ) self.assertEqual(copied, 5) From e7671e14bf6b24d114ae9793d7d54fe08319d3f8 Mon Sep 17 00:00:00 2001 From: Hook25 Date: Mon, 14 Jul 2025 14:24:26 +0200 Subject: [PATCH 3/3] Check that we aren't setting the recepie to buidl for outadted --- tools/release/lp_update_recipe.py | 53 +++++++++++++++++++------- tools/release/test_lp_update_recipe.py | 27 +++++++++++++ 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/tools/release/lp_update_recipe.py b/tools/release/lp_update_recipe.py index 472c5f9d8f..58f3db93e5 100755 --- a/tools/release/lp_update_recipe.py +++ b/tools/release/lp_update_recipe.py @@ -29,19 +29,44 @@ import textwrap import argparse -from utils import get_source_build_recipe - -# this is hardcoded because supporting a new release requires a bit of manual -# effort (copying support packages to the new series and doing the first build -# dance of building packages in dependency order) -active_series = [ - "https://api.launchpad.net/devel/ubuntu/bionic", - "https://api.launchpad.net/devel/ubuntu/focal", - "https://api.launchpad.net/devel/ubuntu/jammy", - "https://api.launchpad.net/devel/ubuntu/noble", - "https://api.launchpad.net/devel/ubuntu/plucky", - "https://api.launchpad.net/devel/ubuntu/questing", -] +from utils import get_source_build_recipe, get_launchpad_client + + +def get_active_series(): + # this is hardcoded because supporting a new release requires a bit of manual + # effort (copying support packages to the new series and doing the first build + # dance of building packages in dependency order) + desired_series = { + "bionic", + "focal", + "jammy", + "noble", + "plucky", + "questing", + } + lp = get_launchpad_client() + active_series = { + series.name for series in lp.projects["ubuntu"].series if series.active + } + + outdated_series = desired_series - active_series + if outdated_series: + print( + "The following series were desired but are now outdated:\n- ", + end="", + ) + print("\n- ".join(outdated_series)) + + undesired_active = active_series - desired_series + if undesired_active: + print( + "The following series weren't desired but are active:\n- ", end="" + ) + print("\n- ".join(undesired_active)) + + possible_series = desired_series & active_series + api_formatter = "https://api.launchpad.net/devel/ubuntu/{}" + return [api_formatter.format(x) for x in possible_series] def get_build_path(recipe_name: str) -> str: @@ -85,7 +110,7 @@ def update_build_recipe( # this is important because while debugging, one may inadvertedly leave a # not-yet-working daily build enabled, this makes the daily builds fail for # no reason - lp_recipe.distroseries = active_series + lp_recipe.distroseries = get_active_series() lp_recipe.lp_save() print(f"Updated build recipe: {lp_recipe.web_link}") print(new_recipe) diff --git a/tools/release/test_lp_update_recipe.py b/tools/release/test_lp_update_recipe.py index 6c000d8bd3..73ad6e7500 100644 --- a/tools/release/test_lp_update_recipe.py +++ b/tools/release/test_lp_update_recipe.py @@ -38,6 +38,10 @@ def test_get_updated_build_recipe(self): @patch("lp_update_recipe.get_source_build_recipe") @patch("lp_update_recipe.print") + @patch( + "lp_update_recipe.get_active_series", + new=MagicMock(return_value="noble"), + ) def test_main(self, print_mock, get_build_path_mock): lp_recipe = get_build_path_mock() @@ -56,3 +60,26 @@ def test_main(self, print_mock, get_build_path_mock): self.assertIn("checkbox-support", lp_recipe.recipe_text) self.assertIn("abcd", lp_recipe.recipe_text) self.assertTrue(lp_recipe.lp_save.called) + + @patch("lp_update_recipe.print") + @patch("lp_update_recipe.get_launchpad_client") + def test_get_active_series(self, get_launchpad_client_mock, print_mock): + lp_mock = get_launchpad_client_mock() + # if this broke, 10+yrs went by and this is still in use. Good :) + # Update the following line with the latest LTS name + active_release_mock = MagicMock(active=True) + active_release_mock.name = "noble" + active_undesired_mock = MagicMock(active=True) + active_undesired_mock.name = " " + lp_mock.projects["ubuntu"].series = [ + active_release_mock, + active_undesired_mock, + ] + returned = lp_update_recipe.get_active_series() + self.assertTrue(any(active_release_mock.name in x for x in returned)) + log = "\n".join(str(x) for x in print_mock.call_args_list) + # active but undesired was reported + self.assertIn(active_undesired_mock.name, log) + # this assumes more than 1 distro is active, if this fails you are + # desiring only 1 distro which is the latest lts from before + self.assertIn("outdated", log)