Upgrade tests

Required PR: pulp/pulpcore#1337

https://pulp.plan.io/issues/8719
closes #8719

.ci/ansible/settings.py.j2

@@ -3,7 +3,7 @@ ANSIBLE_API_HOSTNAME = "http://pulp:80"
 ANSIBLE_CONTENT_HOSTNAME = "http://pulp:80/pulp/content"
 PRIVATE_KEY_PATH = "/etc/pulp/certs/token_private_key.pem"
 PUBLIC_KEY_PATH = "/etc/pulp/certs/token_public_key.pem"
-TOKEN_SERVER = "http://pulp:80/token"
+TOKEN_SERVER = "http://pulp:80/token/"
 TOKEN_SIGNATURE_ALGORITHM = "ES256"
 
 {% if pulp_settings %}

.ci/scripts/cherrypick.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_file' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+set -e
+
+if [ $# -lt 3 ]
+then
+  echo "Usage: .ci/scripts/cherrypick.sh [commit-hash] [original-issue-id] [backport-issue-id]"
+  echo "   ex: .ci/scripts/cherrypick.sh abcd1234 1234 4567"
+  echo ""
+  echo "Note: make sure you are on a fork of the release branch before running this script."
+  exit
+fi
+
+commit="$(git rev-parse $1)"
+issue="$2"
+backport="$3"
+commit_message=$(git log --format=%B -n 1 $commit)
+
+if ! echo $commit_message | tr '[:upper:]' '[:lower:]' | grep -q "\[noissue\]"
+then
+  if ! echo $commit_message | tr '[:upper:]' '[:lower:]' | grep -q -E "(fixes|closes).*#$issue"
+  then
+    echo "Error: issue $issue not detected in commit message." && exit 1
+  fi
+fi
+
+if [ "$4" = "--continue" ]
+then
+  echo "Continue after manually resolving conflicts..."
+elif [ "$4" = "" ]
+then
+  if ! git cherry-pick --no-commit "$commit"
+  then
+    echo "Please resolve and add merge conflicts and restart this command with appended '--continue'."
+    exit 1
+  fi
+else
+  exit 1
+fi
+
+for file in $(find CHANGES -name "$issue.*")
+do
+  newfile="${file/$issue/$backport}"
+  git mv "$file" "$newfile"
+  sed -i -e "\$a (backported from #$issue)" "$newfile"
+  git add "$newfile"
+done
+
+commit_message="$(printf "$commit_message" | sed -E 's/(fixes|closes)/backports/i')"
+commit_message="$commit_message
+
+fixes #$backport
+
+(cherry picked from commit $commit)"
+git commit -m "$commit_message"
+
+printf "\nSuccessfully backported commit $1.\n"

.ci/scripts/publish_client_gem.sh

@@ -12,13 +12,15 @@ set -euv
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../..
 
+export PULP_URL="${PULP_URL:-http://pulp}"
+
 mkdir ~/.gem || true
 touch ~/.gem/credentials
 echo "---
 :rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials
 sudo chmod 600 ~/.gem/credentials
 
-export REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin file -r '.versions[] | select(.component == $plugin) | .version')
+export REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin file --arg legacy_plugin pulp_file -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version')
 export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)"
 if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then
   export VERSION=${REPORTED_VERSION}

.ci/scripts/publish_client_pypi.sh

@@ -9,12 +9,14 @@
 
 set -euv
 
+export PULP_URL="${PULP_URL:-http://pulp}"
+
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../..
 
 pip install twine
 
-export REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin file -r '.versions[] | select(.component == $plugin) | .version')
+export REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin file --arg legacy_plugin pulp_file -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version')
 export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)"
 if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then
   export VERSION=${REPORTED_VERSION}

.ci/scripts/publish_plugin_pypi.sh

@@ -12,8 +12,10 @@ cd "$(dirname "$(realpath -e "$0")")"/../..
 
 set -euv
 
-export VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin file -r '.versions[] | select(.component == $plugin) | .version')
-export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-file-client/$VERSION/)
+export PULP_URL="${PULP_URL:-http://pulp}"
+
+export VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin file --arg legacy_plugin pulp_file -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version')
+export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-file/$VERSION/)
 if [ "$response" == "200" ];
 then
   echo "pulp_file $VERSION has already been released. Skipping."

.ci/scripts/release.py

@@ -6,90 +6,79 @@
 # For more info visit https://github.com/pulp/plugin_template
 
 import argparse
-import json
 import re
 import os
 import textwrap
-from collections import defaultdict
-from pathlib import Path
 
 from git import Repo
+
+from collections import defaultdict
+from pathlib import Path
 from redminelib import Redmine
-from redminelib.exceptions import ResourceAttrError
+from redminelib.exceptions import ResourceAttrError, ResourceSetIndexError
+import json
 
 
 REDMINE_URL = "https://pulp.plan.io"
 REDMINE_QUERY_URL = f"{REDMINE_URL}/issues?set_filter=1&status_id=*&issue_id="
+REDMINE_PROJECT = "pulp_file"
 
 
-def validate_redmine_data(redmine_query_url, redmine_issues):
+def validate_redmine_data(redmine_query_url, redmine_issues, release_version):
     """Validate redmine milestone."""
+    error_messages = []
     redmine = Redmine("https://pulp.plan.io")
-    project_set = set()
+    redmine_project = redmine.project.get(REDMINE_PROJECT)
+    if redmine_project is None:
+        error_messages.append(f"Redmine project {REDMINE_PROJECT} not found.")
+    try:
+        milestone = redmine_project.versions.filter(name=release_version)[0]
+    except ResourceSetIndexError:
+        error_messages.append(f"Milestone {release_version} not found.")
+
     stats = defaultdict(list)
-    milestone_url = "\n[noissue]"
-    milestone_id = None
     for issue in redmine_issues:
         redmine_issue = redmine.issue.get(int(issue))
 
         project_name = redmine_issue.project.name
-        project_set.update([project_name])
         stats[f"project_{project_name.lower().replace(' ', '_')}"].append(issue)
+        if project_name != redmine_project.name:
+            stats["wrong_project"].append(issue)
 
         status = redmine_issue.status.name
         if "CLOSE" not in status and status != "MODIFIED":
             stats["status_not_modified"].append(issue)
 
         try:
-            milestone = redmine_issue.fixed_version.name
-            milestone_id = redmine_issue.fixed_version.id
-            stats[f"milestone_{milestone}"].append(issue)
+            issue_milestone = redmine_issue.fixed_version
+            if redmine_issue.fixed_version.id != milestone.id:
+                stats["wrong_milestone"].append(issue)
+            stats[f"milestone_{issue_milestone.name}"].append(issue)
         except ResourceAttrError:
             stats["without_milestone"].append(issue)
 
-    if milestone_id is not None:
-        milestone_url = f"Redmine Milestone: {REDMINE_URL}/versions/{milestone_id}.json\n[noissue]"
-
     print(f"\n\nRedmine stats: {json.dumps(stats, indent=2)}")
-    error_messages = []
+    if stats.get("wrong_project"):
+        error_messages.append(
+            f"One or more issues are associated to the wrong project {stats['wrong_project']}"
+        )
     if stats.get("status_not_modified"):
         error_messages.append(f"One or more issues are not MODIFIED {stats['status_not_modified']}")
     if stats.get("without_milestone"):
         error_messages.append(
             f"One or more issues are not associated with a milestone {stats['without_milestone']}"
         )
-    if len(project_set) > 1:
-        error_messages.append(f"Issues with different projects - {project_set}")
+    if stats.get("wrong_milestone"):
+        error_messages.append(
+            f"One or more issues are associated to the wrong milestone {stats['wrong_milestone']}"
+        )
+
     if error_messages:
         error_messages.append(f"Verify at {redmine_query_url}")
         raise RuntimeError("\n".join(error_messages))
 
-    return milestone_url
-
+    return f"{milestone.url}.json"
 
-release_path = os.path.dirname(os.path.abspath(__file__))
-plugin_path = release_path.split("/.ci")[0]
-
-plugin_name = "pulp_file"
-version = None
-with open(f"{plugin_path}/setup.py") as fp:
-    for line in fp.readlines():
-        if "version=" in line:
-            version = re.split("\"|'", line)[1]
-    if not version:
-        raise RuntimeError("Could not detect existing version ... aborting.")
-release_version = version.replace(".dev", "")
-
-
-issues_to_close = set()
-for filename in Path(f"{plugin_path}/CHANGES").rglob("*"):
-    if filename.stem.isdigit():
-        issue = filename.stem
-        issues_to_close.add(issue)
-
-issues = ",".join(issues_to_close)
-redmine_final_query = f"{REDMINE_QUERY_URL}{issues}"
-milestone_url = validate_redmine_data(redmine_final_query, issues_to_close)
 
 helper = textwrap.dedent(
     """\
@@ -127,6 +116,31 @@ def validate_redmine_data(redmine_query_url, redmine_issues):
 
 args = parser.parse_args()
 
+release_path = os.path.dirname(os.path.abspath(__file__))
+plugin_path = release_path.split("/.ci")[0]
+
+plugin_name = "pulp_file"
+version = None
+with open(f"{plugin_path}/setup.py") as fp:
+    for line in fp.readlines():
+        if "version=" in line:
+            version = re.split("\"|'", line)[1]
+    if not version:
+        raise RuntimeError("Could not detect existing version ... aborting.")
+release_version = version.replace(".dev", "")
+
+
+issues_to_close = set()
+for filename in Path(f"{plugin_path}/CHANGES").rglob("*"):
+    if filename.stem.isdigit():
+        issue = filename.stem
+        issues_to_close.add(issue)
+
+issues = ",".join(issues_to_close)
+redmine_final_query = f"{REDMINE_QUERY_URL}{issues}"
+milestone_url = validate_redmine_data(redmine_final_query, issues_to_close, release_version)
+
+
 release_type = args.release_type
 
 if "pulpcore" not in release_path:
@@ -164,11 +178,21 @@ def validate_redmine_data(redmine_query_url, redmine_issues):
 git.add(f"{plugin_path}/setup.py")
 git.add(f"{plugin_path}/requirements.txt")
 git.add(f"{plugin_path}/.bumpversion.cfg")
+
 git.commit(
     "-m",
-    f"Release {release_version}\n\nRedmine Query: {redmine_final_query}\n{milestone_url}",
+    "\n".join(
+        [
+            f"Release {release_version}",
+            "",
+            f"Redmine Query: {redmine_final_query}",
+            f"Redmine Milestone: {milestone_url}",
+            "[noissue]",
+        ]
+    ),
 )
 
+
 sha = repo.head.object.hexsha
 short_sha = git.rev_parse(sha, short=7)
 
@@ -198,6 +222,8 @@ def validate_redmine_data(redmine_query_url, redmine_issues):
 git.add(f"{plugin_path}/.bumpversion.cfg")
 git.commit("-m", f"Bump to {new_dev_version}\n\n[noissue]")
 
+
 print(f"\n\nRedmine query of issues to close:\n{redmine_final_query}")
+
 print(f"Release commit == {short_sha}")
 print(f"All changes were committed on branch: release_{release_version}")

.ci/scripts/validate_commit_message.py

@@ -6,16 +6,21 @@
 # For more info visit https://github.com/pulp/plugin_template
 
 import re
-import requests
 import subprocess
 import sys
 from pathlib import Path
 
-KEYWORDS = ["fixes", "closes", "re", "ref"]
+
+import requests
+
+
 NO_ISSUE = "[noissue]"
+CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"]
+
+
+KEYWORDS = ["fixes", "closes", "re", "ref"]
 STATUSES = ["NEW", "ASSIGNED", "POST", "MODIFIED"]
 REDMINE_URL = "https://pulp.plan.io"
-CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"]
 
 sha = sys.argv[1]
 project = "pulp_file"