Better content of backport packages CHANGELOG and INSTALL files (#9013)

* Better content of backport packages CHANGELOG and INSTALL files

The content of Backport Packages CHANGELOG.txt and INSTALL files
has been updated to reflect that those are not full Airflow
releases.

1) Source package:
- INSTALL contains only references to preparing backport packages
- CHANGELOG.txt contains combined change log of all the packages

2) Binary packages:
- No INSTALL
- CHANGELOG.txt contains changelog for this package only

3) Whl packages

No change

* Update backport_packages/INSTALL

.gitignore

@@ -186,3 +186,4 @@ dmypy.json
 /.kube
 /.inputrc
 log.txt*
+/backport_packages/CHANGELOG.txt

backport_packages/INSTALL

@@ -0,0 +1,15 @@
+## INSTALL / BUILD instructions for Apache Airflow Backport packages
+
+NOTE! Those sources are only intended to be used to build Apache Airflow Backport packages, not the Apache Airflow. They have been generated using the unreleased version of Apache Airflow.
+(from master)
+
+This ia a generic installation method that requires docker and docker-compose to be installed.
+
+# [required] fetch the tarball and untar the sources. Move into the directory that was un-tarred.
+
+# [optional] run Apache RAT (release audit tool) to validate license headers
+# RAT docs here: https://creadur.apache.org/rat/. Requires Java and Apache Rat
+java -jar apache-rat.jar -E ./.rat-excludes -d .
+
+The only prerequisites to build the packages is docker and docker-compose.
+The instructions on how to build the packages are described in dev/BACKPORT_PACKAGES.md

backport_packages/build_source_package.sh

@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Use this to sign the tar balls generated from
+# python setup.py sdist --formats=gztar
+# ie. sign.sh <my_tar_ball>
+# you will still be required to type in your signing key password
+# or it needs to be available in your keychain
+set -euo pipefail
+
+MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "${MY_DIR}"/..
+
+function check_version() {
+  if [[ ${VERSION:=} == "" ]]; then
+    echo
+    echo "Please export VERSION variable with the version of source package to prepare"
+    echo
+    exit 1
+  fi
+}
+
+function tag_release() {
+  echo
+  echo "Tagging the sources with backport-providers-${VERSION} tag"
+  echo
+
+  git tag "backport-providers-${VERSION}"
+}
+
+function clean_repo() {
+  ./confirm "Cleaning the repository sources - that might remove some of your unchanged files"
+
+  git clean -fxd
+}
+
+
+function prepare_combined_changelog() {
+  echo
+  echo "Preparing the changelog"
+  echo
+  CHANGELOG_FILE="backport_packages/CHANGELOG.txt"
+  PATTERN="airflow\/providers\/(.*)\/PROVIDERS_CHANGES_.*.md"
+  echo > "${CHANGELOG_FILE}"
+  CHANGES_FILES=$(find "airflow/providers/" -name 'PROVIDERS_CHANGES_*.md' | sort -r)
+  LAST_PROVIDER_ID=""
+  for FILE in ${CHANGES_FILES}
+  do
+      echo "Adding ${FILE}"
+      [[ ${FILE} =~ ${PATTERN} ]]
+      PROVIDER_ID=${BASH_REMATCH[1]//\//.}
+      {
+          if [[ ${LAST_PROVIDER_ID} != "${PROVIDER_ID}" ]]; then
+              echo
+              echo "Provider: ${BASH_REMATCH[1]//\//.}"
+              echo
+              LAST_PROVIDER_ID=${PROVIDER_ID}
+          else
+              echo
+          fi
+          cat "${FILE}"
+          echo
+      } >> "${CHANGELOG_FILE}"
+  done
+
+
+  echo
+  echo "Changelog prepared in ${CHANGELOG_FILE}"
+  echo
+}
+
+function prepare_archive(){
+  echo
+  echo "Preparing the archive ${ARCHIVE_FILE_NAME}"
+  echo
+
+  git archive \
+      --format=tar.gz \
+      "backport-providers-${VERSION}" \
+      "--prefix=apache-airflow-backport-providers-${VERSION}/" \
+      -o "${ARCHIVE_FILE_NAME}"
+
+  echo
+  echo "Prepared the archive ${ARCHIVE_FILE_NAME}"
+  echo
+
+}
+
+
+function replace_install_changelog(){
+  DIR=$(mktemp -d)
+
+  echo
+  echo "Replacing INSTALL CHANGELOG.txt in ${ARCHIVE_FILE_NAME} "
+  echo
+  tar -f "apache-airflow-backport-providers-${VERSION}-source.tar.gz" -xz -C "${DIR}"
+
+  cp "backport_packages/INSTALL" "backport_packages/CHANGELOG.txt" \
+      "${DIR}/apache-airflow-backport-providers-${VERSION}/"
+
+  tar -f "apache-airflow-backport-providers-${VERSION}-source.tar.gz" -cz -C "${DIR}" \
+      "apache-airflow-backport-providers-${VERSION}/"
+
+  echo
+  echo "Replaced INSTALL CHANGELOG.txt in ${ARCHIVE_FILE_NAME} "
+  echo
+
+}
+
+
+
+check_version
+
+export ARCHIVE_FILE_NAME="apache-airflow-backport-providers-${VERSION}-source.tar.gz"
+
+tag_release
+clean_repo
+prepare_archive
+prepare_combined_changelog
+replace_install_changelog

dev/BACKPORT_PACKAGES.md

@@ -253,27 +253,14 @@ export AIRFLOW_REPO_ROOT=$(pwd)
 
 ```
 
-* Tag the release
+* Build the source package:
 
-```bash
-git tag backport-providers-${VERSION}
 ```
+./backport_packages/build_source_package.sh
 
-* Clean the checkout
-
-```bash
-git clean -fxd
 ```
 
-* Tarball the sources
-
-```bash
-git archive \
-    --format=tar.gz \
-    backport-providers-${VERSION} \
-    --prefix=apache-airflow-backport-providers-${VERSION}/ \
-    -o apache-airflow-backport-providers-${VERSION}-source.tar.gz
-```
+It will generate `apache-airflow-backport-providers-${VERSION}-source.tar.gz`
 
 * Generate the packages - since we are preparing packages for SVN repo, we should use the right switch. Note
   that this will clean up dist folder before generating the packages, so it will only contain the packages

scripts/ci/in_container/run_prepare_backport_packages.sh

@@ -105,6 +105,42 @@ do
         cat "${LOG_FILE}"
         exit "${RES}"
     fi
+    echo > "${LOG_FILE}"
+
+    PACKAGE_DIR=${BACKPORT_PACKAGE//./\/}
+
+    PATTERN="airflow\/providers\/(.*)\/PROVIDERS_CHANGES_.*.md"
+    CHANGELOG_FILE="CHANGELOG.txt"
+
+    echo > "${CHANGELOG_FILE}"
+    CHANGES_FILES=$(find "airflow/providers/${PACKAGE_DIR}" -name 'PROVIDERS_CHANGES_*.md' | sort -r)
+    LAST_PROVIDER_ID=""
+    for FILE in ${CHANGES_FILES}
+    do
+        [[ ${FILE} =~ ${PATTERN} ]]
+        PROVIDER_ID=${BASH_REMATCH[1]//\//.}
+        {
+            if [[ ${LAST_PROVIDER_ID} != "${PROVIDER_ID}" ]]; then
+                echo
+                echo "Provider: ${BASH_REMATCH[1]//\//.}"
+                echo
+                LAST_PROVIDER_ID=${PROVIDER_ID}
+            else
+                echo
+            fi
+            cat "${FILE}"
+            echo
+        } >> "${CHANGELOG_FILE}"
+    done
+
+    echo "Changelog prepared in ${CHANGELOG_FILE} for ${PACKAGE_DIR}"
+    set +e
+    python3 setup_backport_packages.py "${BACKPORT_PACKAGE}" clean --all >"${LOG_FILE}" 2>&1
+    RES="${?}"
+    if [[ ${RES} != "0" ]]; then
+        cat "${LOG_FILE}"
+        exit "${RES}"
+    fi
     python3 setup_backport_packages.py --version-suffix "${VERSION_SUFFIX_FOR_PYPI}     " \
         "${BACKPORT_PACKAGE}" sdist bdist_wheel >"${LOG_FILE}" 2>&1
     RES="${?}"
@@ -114,6 +150,7 @@ do
     fi
     set -e
     echo " Prepared backport package ${BACKPORT_PACKAGE}"
+    echo "==================================================================================="
 done
 
 cd "${AIRFLOW_SOURCES}" || exit 1