Merge pull request #117 from ATIX-AG/update_ci

Update CI

.ci/scripts/collect_changes.py

@@ -1,13 +1,16 @@
+#!/bin/env python3
+
 import itertools
 import os
 import re
+import tomllib
 
-import toml
 from git import GitCommandError, Repo
 from packaging.version import parse as parse_version
 
 # Read Towncrier settings
-tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"]
+with open("pyproject.toml", "rb") as fp:
+    tc_settings = tomllib.load(fp)["tool"]["towncrier"]
 
 CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst")
 START_STRING = tc_settings.get(
@@ -21,23 +24,32 @@
 TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})")
 
 
+# Build a regex to find the header of a changelog section.
+# It must have a single capture group to single out the version.
+# see help(re.split) for more info.
 NAME_REGEX = r".*"
-VERSION_REGEX = r"([0-9]+\.[0-9]+\.[0-9][0-9ab]*)"
+VERSION_REGEX = r"[0-9]+\.[0-9]+\.[0-9][0-9ab]*"
+VERSION_CAPTURE_REGEX = rf"({VERSION_REGEX})"
 DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}"
 TITLE_REGEX = (
     "("
     + re.escape(
         TITLE_FORMAT.format(name="NAME_REGEX", version="VERSION_REGEX", project_date="DATE_REGEX")
     )
     .replace("NAME_REGEX", NAME_REGEX)
+    .replace("VERSION_REGEX", VERSION_CAPTURE_REGEX, 1)
     .replace("VERSION_REGEX", VERSION_REGEX)
     .replace("DATE_REGEX", DATE_REGEX)
     + ")"
 )
 
 
 def get_changelog(repo, branch):
-    return repo.git.show(f"{branch}:{CHANGELOG_FILE}") + "\n"
+    branch_tc_settings = tomllib.loads(repo.git.show(f"{branch}:pyproject.toml"))["tool"][
+        "towncrier"
+    ]
+    branch_changelog_file = branch_tc_settings.get("filename", "NEWS.rst")
+    return repo.git.show(f"{branch}:{branch_changelog_file}") + "\n"
 
 
 def _tokenize_changes(splits):
@@ -89,7 +101,7 @@ def main():
             for change in main_changes:
                 fp.write(change[1])
 
-        repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE)
+        repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE)
 
 
 if __name__ == "__main__":

.ci/scripts/create_release_branch.sh

@@ -10,7 +10,7 @@ then
   exit 1
 fi
 
-NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')"
+NEW_BRANCH="$(bump-my-version show new_version --increment release | sed -Ene 's/^([[:digit:]]+\.[[:digit:]]+)\.[[:digit:]]+$/\1/p')"
 
 if [[ -z "${NEW_BRANCH}" ]]
 then
@@ -23,6 +23,6 @@ git branch "${NEW_BRANCH}"
 # Clean changelog snippets.
 find CHANGES/ \( -name "*.feature" -o -name "*.bugfix" -o -name "*.removal" -o -name "*.doc" -o -name "*.translation" -o -name "*.devel" -o -name "*.misc" \) -exec git rm -f \{\} +
 
-bump2version minor --commit --message $'Bump version to {new_version}\n\n[noissue]' --allow-dirty
+bump-my-version bump minor --commit --message $'Bump version to {new_version}' --allow-dirty
 
 git push origin "${NEW_BRANCH}"

.ci/scripts/pr_labels.py

@@ -0,0 +1,57 @@
+#!/bin/env python3
+
+# This script is running with elevated privileges from the main branch against pull requests.
+
+import re
+import sys
+import tomllib
+from pathlib import Path
+
+from git import Repo
+
+
+def main():
+    assert len(sys.argv) == 3
+
+    with open("pyproject.toml", "rb") as fp:
+        PYPROJECT_TOML = tomllib.load(fp)
+    BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT")
+    ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)")
+    CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$")
+    CHANGELOG_EXTS = [
+        f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"]
+    ]
+
+    repo = Repo(".")
+
+    base_commit = repo.commit(sys.argv[1])
+    head_commit = repo.commit(sys.argv[2])
+
+    pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}"))
+
+    labels = {
+        "multi-commit": len(pr_commits) > 1,
+        "cherry-pick": False,
+        "no-issue": False,
+        "no-changelog": False,
+        "wip": False,
+    }
+    for commit in pr_commits:
+        labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None
+        no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None
+        labels["no-issue"] |= no_issue
+        cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None
+        labels["cherry-pick"] |= cherry_pick
+        changelog_snippets = [
+            k
+            for k in commit.stats.files
+            if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS
+        ]
+        labels["no-changelog"] |= not changelog_snippets
+
+    print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v)))
+    print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v)))
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/release.sh

@@ -10,7 +10,7 @@ then
   exit 1
 fi
 
-NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')"
+NEW_VERSION="$(bump-my-version show new_version --increment release)"
 echo "Release ${NEW_VERSION}"
 
 if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]]
@@ -20,7 +20,7 @@ then
 fi
 
 towncrier build --yes --version "${NEW_VERSION}"
-bump2version release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty
-bump2version patch --commit
+bump-my-version bump release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty
+bump-my-version bump patch --commit
 
 git push origin "${BRANCH}" "${NEW_VERSION}"

.ci/scripts/validate_commit_message.py

@@ -2,11 +2,13 @@
 import re
 import subprocess
 import sys
+import tomllib
 from pathlib import Path
 
-import toml
 from github import Github
 
+with open("pyproject.toml", "rb") as fp:
+    PYPROJECT_TOML = tomllib.load(fp)
 KEYWORDS = ["fixes", "closes"]
 BLOCKING_REGEX = [
     "DRAFT",
@@ -15,10 +17,7 @@
     r"DO\s*NOT\s*MERGE",
     "EXPERIMENT",
 ]
-NO_ISSUE = "[noissue]"
-CHANGELOG_EXTS = [
-    f".{item['directory']}" for item in toml.load("pyproject.toml")["tool"]["towncrier"]["type"]
-]
+CHANGELOG_EXTS = [f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"]]
 
 sha = sys.argv[1]
 message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8")
@@ -61,13 +60,5 @@ def check_changelog(issue):
         if not cherry_pick:
             check_status(issue)
             check_changelog(issue)
-else:
-    if NO_ISSUE in message:
-        print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE))
-    else:
-        sys.exit(
-            "Error: no attached issues found for {sha}. If this was intentional, add "
-            " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE)
-        )
 
 print("Commit message for {sha} passed.".format(sha=sha[0:7]))

.github/dependabot.yml

@@ -5,12 +5,8 @@ updates:
   schedule:
     interval: daily
   open-pull-requests-limit: 10
-  commit-message:
-    prefix: "[noissue]"
 - package-ecosystem: github-actions
   directory: "/"
   schedule:
     interval: daily
   open-pull-requests-limit: 10
-  commit-message:
-    prefix: "[noissue]"

.github/workflows/collect_changes.yml

@@ -20,7 +20,7 @@ jobs:
           git config user.email pulp-infra@redhat.com
       - name: "Collect changes"
         run: |
-          pip install GitPython packaging toml
+          pip install GitPython packaging
           python3 .ci/scripts/collect_changes.py
       - name: "Create Pull Request"
         uses: "peter-evans/create-pull-request@v6"

.github/workflows/pr_checks.yml

@@ -15,25 +15,31 @@ concurrency:
 jobs:
   single_commit:
     runs-on: "ubuntu-latest"
-    name: "Label multiple commit PR"
+    name: "Label PR"
     permissions:
       pull-requests: "write"
     steps:
       - uses: "actions/checkout@v4"
         with:
           fetch-depth: 0
-      - name: "Commit Count Check"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: "Determine PR labels"
         run: |
+          pip install GitPython==3.1.42
           git fetch origin ${{ github.event.pull_request.head.sha }}
-          echo "COMMIT_COUNT=$(git log  --oneline --no-merges origin/${{ github.base_ref }}..${{ github.event.pull_request.head.sha }} | wc -l)" >> "$GITHUB_ENV"
+          python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV"
       - uses: "actions/github-script@v7"
+        name: "Apply PR Labels"
         with:
           script: |
-            const labelName = "multi-commit";
-            const { COMMIT_COUNT } = process.env;
+            const { ADD_LABELS, REMOVE_LABELS } = process.env;
+
+            const addLabels = ADD_LABELS.split(",");
+            const removeLabels = REMOVE_LABELS.split(",");
 
-            if (COMMIT_COUNT == 1)
-            {
+            for await (const labelName of removeLabels) {
               try {
                 await github.rest.issues.removeLabel({
                   issue_number: context.issue.number,
@@ -44,8 +50,7 @@ jobs:
               } catch(err) {
               }
             }
-            else
-            {
+            for await (const labelName of addLabels) {
               await github.rest.issues.addLabels({
                 issue_number: context.issue.number,
                 owner: context.repo.owner,

.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
       - name: "Install dependencies"
         run: |
           python -m pip install --upgrade pip
-          pip install bump2version towncrier~=23.11.0
+          pip install bump-my-version~=0.20.0 towncrier~=23.11.0
       - name: "Setup git"
         run: |
           git config user.name pulpbot

.github/workflows/release_branch.yml

@@ -18,7 +18,7 @@ jobs:
           git config user.email pulp-infra@redhat.com
       - name: "Install python dependencies"
         run: |
-          pip install bump2version
+          pip install bump-my-version~=0.20.0
       - name: "Create Release Branch"
         run: |
           .ci/scripts/create_release_branch.sh

.github/workflows/test.yml

@@ -16,21 +16,21 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - python: "3.9"
-            image_tag: "nightly"
+          - image_tag: "nightly"
             pulp_api_root: "/relocated/djnd/"
-          - python: "3.8"
-            image_tag: "3.21"
-          - python: "3.12"
-            image_tag: "3.20"
+            python: "3.9"
+          - image_tag: "3.21"
+            python: "3.8"
+          - image_tag: "3.20"
             lower_bounds: true
-          - python: "3.8"
-            image_tag: "3.19"
-          - python: "3.11"
-            image_tag: "3.18"
-          - python: "3.10"
-            image_tag: "3.22"
+            python: "3.12"
+          - image_tag: "3.19"
+            python: "3.8"
+          - image_tag: "3.18"
+            python: "3.11"
+          - image_tag: "3.22"
             lower_bounds: true
+            python: "3.10"
     steps:
       - uses: "actions/checkout@v4"
       - uses: "actions/cache@v4"
@@ -48,7 +48,7 @@ jobs:
         uses: "actions/setup-python@v5"
         with:
           python-version: "${{ matrix.python }}"
-      - name: "Install Test Dependencies"
+      - name: "Install Python Test Dependencies"
         run: |
           if [ "${{matrix.lower_bounds}}" ]
           then

.gitignore

@@ -2,5 +2,8 @@
 __pycache__/
 build/
 tests/cli.toml
-/.ci/settings/
-!/.ci/settings/settings.py
+pytest_pulp_cli/GPG-PRIVATE-KEY-fixture-signing
+/.ci/settings/certs
+site/
+dist/
+*.po~

CHANGES/.TEMPLATE.md

@@ -1,13 +1,16 @@
 
 {# TOWNCRIER TEMPLATE #}
 {% for section, _ in sections.items() %}
-{% if section %}### {{section}}
+{%- set section_slug = "-" + section|replace(" ", "-")|replace("_", "-")|lower %}
+{% if section %}### {{section}} {: #{{versiondata.version}}{{section_slug}} }
 
+{% else %}
+{%- set section_slug = "" %}
 {% endif %}
 
 {% if sections[section] %}
 {% for category, val in definitions.items() if category in sections[section]%}
-#### {{ definitions[category]['name'] }}
+#### {{ definitions[category]['name'] }} {: #{{versiondata.version}}{{section_slug}}-{{category}} }
 
 {% if definitions[category]['showcontent'] %}
 {% for text, values in sections[section][category].items() %}