Automate rcN calculation when releasing provider packages (#36441)

This change will automatically generate the **right** rcN package
when prepareing the packages for PyPI. This allows to have pretty
much continuous release process for voting over the provider packages.

Simply when an rcN candidate is not released, it will be automatically
included in the next wave of packages with rcN+1 version - unless during
provider package generation the version will be bumped to MAJOR or MINOR
due to new changes.

This allows for the workflow where in every new wave we always generate
all provider packages ready for release.

dev/README_RELEASE_PROVIDER_PACKAGES.md

@@ -206,7 +206,7 @@ are not generated. Release notes are only generated, when the latest version of
 yet have a corresponding TAG.
 
 The tags for providers is of the form ``providers-<PROVIDER_ID>/<VERSION>`` for example
-``providers-amazon/1.0.0``. During releasing, the RC1/RC2 tags are created (for example
+``providers-amazon/1.0.0``. During releasing, the `rc*` tags are created (for example
 ``providers-amazon/1.0.0rc1``).
 
 Details about maintaining the SEMVER version are going to be discussed and implemented in
@@ -372,8 +372,8 @@ so you need to use `--version-suffix-for-pypi` switch to prepare those packages.
 Note that these are different packages than the ones used for SVN upload
 though they should be generated from the same sources.
 
-* Generate the packages with the right RC version (specify the version suffix with PyPI switch). Note that
-this will clean up dist folder before generating the packages, so you will only have the right packages there.
+* Generate the packages with the rc1 version (specify the version suffix with PyPI switch). Note that
+you should clean up dist folder before generating the packages, so you will only have the right packages there.
 
 ```shell script
 rm -rf ${AIRFLOW_REPO_ROOT}/dist/*
@@ -389,13 +389,10 @@ breeze release-management prepare-provider-packages \
 --version-suffix-for-pypi rc1 --package-format both PACKAGE PACKAGE ....
 ```
 
-In case you are ALSO releasing RC2, RC3, etc. for selected packages, they will be skipped automatically because
-the `rc1` tag will be created for them already. In this case you should specify the ``rc*`` that you want to
-build and specify the package id's you want to build.
-
-```shell script
-breeze release-management prepare-provider-packages --version-suffix-for-pypi rc2 --package-format both PACKAGE
-```
+In case some packages already had rc1 suffix prepared and released, and they still need to be released, they
+will have automatically appropriate rcN suffix added to them. The suffix will be increased for each release
+candidate and checked if tag has been already created for that release candidate. If yes, the suffix will be
+increased until the tag is not found.
 
 * Verify the artifacts that would be uploaded:
 

dev/breeze/src/airflow_breeze/commands/release_management_commands.py

@@ -564,17 +564,20 @@ def prepare_provider_packages(
         shutil.rmtree(DIST_DIR, ignore_errors=True)
         DIST_DIR.mkdir(parents=True, exist_ok=True)
     for provider_id in packages_list:
+        package_version = version_suffix_for_pypi
         try:
             basic_provider_checks(provider_id)
-            if not skip_tag_check and should_skip_the_package(provider_id, version_suffix_for_pypi):
-                continue
+            if not skip_tag_check:
+                should_skip, package_version = should_skip_the_package(provider_id, package_version)
+                if should_skip:
+                    continue
             get_console().print()
             with ci_group(f"Preparing provider package [special]{provider_id}"):
                 get_console().print()
                 target_provider_root_sources_path = copy_provider_sources_to_target(provider_id)
                 generate_build_files(
                     provider_id=provider_id,
-                    version_suffix=version_suffix_for_pypi,
+                    version_suffix=package_version,
                     target_provider_root_sources_path=target_provider_root_sources_path,
                 )
                 cleanup_build_remnants(target_provider_root_sources_path)

dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py

@@ -155,19 +155,30 @@ def generate_build_files(provider_id: str, version_suffix: str, target_provider_
     get_console().print(f"\n[info]Generated package build files for {provider_id}[/]\n")
 
 
-def should_skip_the_package(provider_id: str, version_suffix: str) -> bool:
-    """Return True if the package should be skipped.
+def should_skip_the_package(provider_id: str, version_suffix: str) -> tuple[bool, str]:
+    """Return True, version if the package should be skipped and False, good version suffix if not.
 
     For RC and official releases we check if the "officially released" version exists
     and skip the released if it was. This allows to skip packages that have not been
     marked for release in this wave. For "dev" suffixes, we always build all packages.
     """
-    if version_suffix.startswith("rc") or version_suffix == "":
-        current_tag = get_latest_provider_tag(provider_id, version_suffix)
+    if version_suffix != "" and not version_suffix.startswith("rc"):
+        return False, version_suffix
+    if version_suffix == "":
+        current_tag = get_latest_provider_tag(provider_id, "")
         if tag_exists_for_provider(provider_id, current_tag):
-            get_console().print(f"[warning]The tag {current_tag} exists. Skipping the package.[/]")
-            return True
-    return False
+            get_console().print(f"[warning]The 'final' tag {current_tag} exists. Skipping the package.[/]")
+            return True, version_suffix
+        return False, version_suffix
+    # version_suffix starts with "rc"
+    current_version = int(version_suffix[2:])
+    while True:
+        current_tag = get_latest_provider_tag(provider_id, f"rc{current_version}")
+        if tag_exists_for_provider(provider_id, current_tag):
+            current_version += 1
+            get_console().print(f"[warning]The tag {current_tag} exists. Checking rc{current_version}.[/]")
+        else:
+            return False, f"rc{current_version}"
 
 
 def cleanup_build_remnants(target_provider_root_sources_path: Path):