Fix errors in common.io release documentation generation with prevent…

…ion (#35236)

The common.io provider release caused a few problems when preparing the
new wave of the providers:

* it missed link to security.rst (which is generated automatically
  anticipating SBOM generation in the near future)
* it did not have proper min-version set in
  MIN_AIRFLOW_VERSION_EXCEPTIONS
* it had single-backtick description that resulted in generated commit
  and changelog code - since it failed documentation building

All those errors have not been caught at PR time. This PR fixes them
and introduces prevention from similar mistakes - future providers with
similar errors will get nice errors generated in CI at "docs publishing
step".

* if some generated file is missing in the index user is asked to add
  the files in the index
* if there are `backtick` errors they will be reported and pointed at
* if there are liniting problems with generated .rst files they will
  be reported
* the "min airflow version" is now taken directly from provider.yaml
  not from exception list - but only when it is bigger than current
  min version. This way, we will also not forget to remove the provider
  from the list of exceptions when we bump min version above the
  exception version

airflow/providers/common/io/__init__.py

@@ -1,3 +1,4 @@
+#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,3 +15,29 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+#
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
+# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES.
+#
+# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
+# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/provider_packages` DIRECTORY
+#
+from __future__ import annotations
+
+import packaging.version
+
+__all__ = ["__version__"]
+
+__version__ = "1.0.0"
+
+try:
+    from airflow import __version__ as airflow_version
+except ImportError:
+    from airflow.version import version as airflow_version
+
+if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
+    "2.8.0"
+):
+    raise RuntimeError(
+        f"The package `apache-airflow-providers-common-io:{__version__}` requires Apache Airflow 2.8.0+"
+    )

airflow/providers/common/io/provider.yaml

@@ -19,7 +19,7 @@
 package-name: apache-airflow-providers-common-io
 name: Common IO
 description: |
-  `Common IO Provider`
+  ``Common IO Provider``
 
 suspended: false
 versions:

dev/provider_packages/prepare_provider_packages.py

@@ -55,10 +55,6 @@
 ALL_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11"]
 
 MIN_AIRFLOW_VERSION = "2.5.0"
-# In case you have some providers that you want to have different min-airflow version for,
-# Add them as exceptions here. Make sure to remove it once the min-airflow version is bumped
-# to the same version that is required by the exceptional provider
-MIN_AIRFLOW_VERSION_EXCEPTIONS = {"openlineage": "2.7.0"}
 
 INITIAL_CHANGELOG_CONTENT = """
  .. Licensed to the Apache Software Foundation (ASF) under one
@@ -1122,8 +1118,11 @@ def get_provider_jinja_context(
     for p in provider_details.excluded_python_versions:
         python_requires += f", !={p}"
     min_airflow_version = MIN_AIRFLOW_VERSION
-    if MIN_AIRFLOW_VERSION_EXCEPTIONS.get(provider_details.provider_package_id):
-        min_airflow_version = MIN_AIRFLOW_VERSION_EXCEPTIONS[provider_details.provider_package_id]
+    for dependency in provider_info["dependencies"]:
+        if dependency.startswith("apache-airflow>="):
+            current_min_airflow_version = dependency.split(">=")[1]
+            if Version(current_min_airflow_version) > Version(min_airflow_version):
+                min_airflow_version = current_min_airflow_version
     context: dict[str, Any] = {
         "ENTITY_TYPES": list(EntityType),
         "README_FILE": "README.rst",
@@ -1335,25 +1334,32 @@ def update_release_notes(
     )
     jinja_context["DETAILED_CHANGES_RST"] = changes
     jinja_context["DETAILED_CHANGES_PRESENT"] = bool(changes)
-    update_changelog_rst(
+    errors = False
+    if not update_changelog_rst(
         jinja_context,
         provider_package_id,
         provider_details.documentation_provider_package_path,
         regenerate_missing_docs,
-    )
-    update_security_rst(
+    ):
+        errors = True
+    if not update_security_rst(
         jinja_context,
         provider_package_id,
         provider_details.documentation_provider_package_path,
         regenerate_missing_docs,
-    )
+    ):
+        errors = True
     if not force:
-        update_commits_rst(
+        if not update_commits_rst(
             jinja_context,
             provider_package_id,
             provider_details.documentation_provider_package_path,
             regenerate_missing_docs,
-        )
+        ):
+            errors = True
+    if errors:
+        console.print("[red]There were errors when generating documentation[/]")
+        sys.exit(1)
     return True
 
 
@@ -1453,6 +1459,11 @@ def update_index_rst(
     replace_content(index_file_path, old_text, new_text, provider_package_id)
 
 
+# Taken from pygrep hooks we are using in pre-commit
+# https://github.com/pre-commit/pygrep-hooks/blob/main/.pre-commit-hooks.yaml
+BACKTICKS_CHECK = re.compile(r"^(?!    ).*(^| )`[^`]+`([^_]|$)", re.MULTILINE)
+
+
 def _update_file(
     context: dict[str, Any],
     template_name: str,
@@ -1461,10 +1472,10 @@ def _update_file(
     provider_package_id: str,
     target_path: Path,
     regenerate_missing_docs: bool,
-):
+) -> bool:
     file_path = target_path / file_name
     if regenerate_missing_docs and file_path.exists():
-        return
+        return True
     new_text = render_template(
         template_name=template_name, context=context, extension=extension, keep_trailing_newline=True
     )
@@ -1474,15 +1485,62 @@ def _update_file(
         with open(file_path) as readme_file_read:
             old_text = readme_file_read.read()
     replace_content(file_path, old_text, new_text, provider_package_id)
+    index_path = target_path / "index.rst"
+    if not index_path.exists():
+        console.print(f"[red]ERROR! The index must exist for the provider docs: {index_path}")
+        sys.exit(1)
+
+    expected_link_in_index = f"<{file_name.split('.')[0]}>"
+    if expected_link_in_index not in index_path.read_text():
+        console.print(
+            f"\n[red]ERROR! The {index_path} must contain "
+            f"link to the generated documentation:[/]\n\n"
+            f"[yellow]{expected_link_in_index}[/]\n\n"
+            f"[bright_blue]Please make sure to add it to {index_path}.\n"
+        )
+
+    console.print(f"Checking for backticks correctly generated in: {file_path}")
+    match = BACKTICKS_CHECK.search(file_path.read_text())
+    if match:
+        console.print(
+            f"\n[red]ERROR: Single backticks (`) found in {file_path}:[/]\n\n"
+            f"[yellow]{match.group(0)}[/]\n\n"
+            f"[bright_blue]Please fix them by replacing with double backticks (``).[/]\n"
+        )
+        return False
+
+    # TODO: uncomment me. Linting revealed that our already generated provider docs have duplicate links
+    #       in the generated files, we should fix those and uncomment linting as separate step - so that
+    #       we do not hold current release for fixing the docs.
+    # console.print(f"Linting: {file_path}")
+    # errors = restructuredtext_lint.lint_file(file_path)
+    # real_errors = False
+    # if errors:
+    #     for error in errors:
+    #         # Skip known issue: linter with doc role similar to https://github.com/OCA/pylint-odoo/issues/38
+    #         if (
+    #             'No role entry for "doc"' in error.message
+    #             or 'Unknown interpreted text role "doc"' in error.message
+    #         ):
+    #             continue
+    #         real_errors = True
+    #         console.print(f"* [red] {error.message}")
+    #     if real_errors:
+    #         console.print(f"\n[red] Errors found in {file_path}")
+    #         return False
+
+    console.print(f"[green]Generated {file_path} for {provider_package_id} is OK[/]")
+
+    return True
 
 
 def update_changelog_rst(
     context: dict[str, Any],
     provider_package_id: str,
     target_path: Path,
     regenerate_missing_docs: bool,
-):
-    _update_file(
+) -> bool:
+    return _update_file(
         context=context,
         template_name="PROVIDER_CHANGELOG",
         extension=".rst",
@@ -1498,8 +1556,8 @@ def update_security_rst(
     provider_package_id: str,
     target_path: Path,
     regenerate_missing_docs: bool,
-):
-    _update_file(
+) -> bool:
+    return _update_file(
         context=context,
         template_name="PROVIDER_SECURITY",
         extension=".rst",
@@ -1515,8 +1573,8 @@ def update_commits_rst(
     provider_package_id: str,
     target_path: Path,
     regenerate_missing_docs: bool,
-):
-    _update_file(
+) -> bool:
+    return _update_file(
         context=context,
         template_name="PROVIDER_COMMITS",
         extension=".rst",

docs/apache-airflow-providers-common-io/commits.rst

@@ -16,21 +16,13 @@
     specific language governing permissions and limitations
     under the License.
 
+.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME!
 
 Package apache-airflow-providers-common-io
-------------------------------------------
+------------------------------------------------------
+
+``Common IO Provider``
 
-Common IO Provider
 
 This is detailed commit list of changes for versions provider package: ``common.io``.
 For high-level changelog, see :doc:`package information including changelog <index>`.
-
-1.0.0
-.....
-
-Latest change: 2023-10-19
-
-=================================================================================================  ===========  ========================================================
-Commit                                                                                             Committed    Subject
-=================================================================================================  ===========  ========================================================
-=================================================================================================  ===========  ========================================================

docs/apache-airflow-providers-common-io/index.rst

@@ -27,6 +27,7 @@
 
     Home <self>
     Changelog <changelog>
+    Security <security>
 
 .. toctree::
     :hidden:
@@ -68,3 +69,36 @@
     :caption: Commits
 
     Detailed list of commits <commits>
+
+
+Package apache-airflow-providers-common-io
+------------------------------------------------------
+
+``Common IO Provider``
+
+
+Release: 1.0.0
+
+Provider package
+----------------
+
+This is a provider package for ``common.io`` provider. All classes for this provider package
+are in ``airflow.providers.common.io`` python package.
+
+Installation
+------------
+
+You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below)
+for the minimum Airflow version supported) via
+``pip install apache-airflow-providers-common-io``
+
+Requirements
+------------
+
+The minimum Apache Airflow version supported by this provider package is ``2.8.0``.
+
+==================  ==================
+PIP package         Version required
+==================  ==================
+``apache-airflow``  ``>=2.8.0``
+==================  ==================

docs/apache-airflow-providers-common-io/security.rst

@@ -0,0 +1,38 @@
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Releasing security patches
+--------------------------
+
+Airflow providers are released independently from Airflow itself and the information about vulnerabilities
+is published separately. You can upgrade providers independently from Airflow itself, following the
+instructions found in :doc:`apache-airflow:installation/installing-from-pypi`.
+
+When we release Provider version, the development is always done from the ``main`` branch where we prepare
+the next version. The provider uses strict `SemVer <https://semver.org>`_ versioning policy. Depending on
+the scope of the change, Provider will get ''MAJOR'' version upgrade when there are
+breaking changes, ``MINOR`` version upgrade when there are new features or ``PATCHLEVEL`` version upgrade
+when there are only bug fixes (including security bugfixes) - and this is the only version that receives
+security fixes by default, so you should upgrade to latest version of the provider if you want to receive
+all released security fixes.
+
+The only exception to that rule is when we have a critical security fix and good reason to provide an
+out-of-band release for the provider, in which case stakeholders in the provider might decide to cherry-pick
+and prepare a branch for an older version of the provider following the
+`mixed governance model <https://github.com/apache/airflow/blob/main/PROVIDERS.rst#mixed-governance-model>`_
+and requires interested parties to cherry-pick and test the fixes.

setup.py

@@ -436,6 +436,7 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve
     "pipdeptree",
     "pygithub",
     "rich-click>=1.7.0",
+    "restructuredtext-lint",
     "semver",
     "towncrier",
     "twine",