Update CI

[noissue]

.ci/ansible/Containerfile.j2

@@ -1,4 +1,4 @@
-FROM {{ ci_base | default("pulp/pulp-ci-centos:latest") }}
+FROM {{ ci_base | default("ghcr.io/pulp/pulp-ci-centos:" + pulp_container_tag) }}
 
 # Add source directories to container
 {% for item in plugins %}
@@ -11,9 +11,15 @@ ADD {{ item.source }} {{ item.source }}
 # Hacking botocore (https://github.com/boto/botocore/pull/1990)
 
 RUN pip3 install \
+{%- if stream_test | default(false) -%}
+{{ " " }}django-storages[sftp]
+{%- endif -%}
 {%- if s3_test | default(false) -%}
 {{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue
 {%- endif -%}
+{%- if azure_test | default(false) -%}
+{{ " " }}django-storages[azure]>=1.12.2
+{%- endif -%}
 {%- for item in plugins -%}
 {%- if item.name == "pulp-certguard" -%}
 {{ " " }}python-dateutil rhsm
@@ -23,7 +29,8 @@ RUN pip3 install \
 
 RUN mkdir -p /etc/nginx/pulp/
 {% for item in plugins %}
-RUN ln /usr/local/lib/python3.6/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true
+RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \
+    ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true
 {% endfor %}
 
 ENTRYPOINT ["/init"]

.ci/ansible/inventory.yaml

@@ -6,6 +6,7 @@ all:
         pulp:
         pulp-fixtures:
         minio:
+        ci-sftp:
       vars:
         ansible_connection: docker
 ...

.ci/ansible/settings.py.j2

@@ -1,18 +1,39 @@
-CONTENT_ORIGIN = "http://pulp:80"
-ANSIBLE_API_HOSTNAME = "http://pulp:80"
-ANSIBLE_CONTENT_HOSTNAME = "http://pulp:80/pulp/content"
+CONTENT_ORIGIN = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}"
+ANSIBLE_API_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}"
+ANSIBLE_CONTENT_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/pulp/content"
 PRIVATE_KEY_PATH = "/etc/pulp/certs/token_private_key.pem"
 PUBLIC_KEY_PATH = "/etc/pulp/certs/token_public_key.pem"
-TOKEN_SERVER = "http://pulp:80/token/"
+TOKEN_SERVER = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/token/"
 TOKEN_SIGNATURE_ALGORITHM = "ES256"
+CACHE_ENABLED = True
+REDIS_HOST = "localhost"
+REDIS_PORT = 6379
+
+{% if api_root is defined %}
+API_ROOT = {{ api_root | repr }}
+{% endif %}
 
 {% if pulp_settings %}
 {% for key, value in pulp_settings.items() %}
 {{ key | upper }} = {{ value | repr }}
 {% endfor %}
 {% endif %}
 
+{% if stream_test | default(false) -%}
+REDIRECT_TO_OBJECT_STORAGE = False
+DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.PulpSFTPStorage"
+MEDIA_ROOT = ""
+SFTP_STORAGE_HOST = "ci-sftp"
+SFTP_STORAGE_ROOT = "/storage/"
+SFTP_STORAGE_PARAMS = {
+    "username": "foo",
+    "key_filename": "/keys/id_ed25519",
+}
+{%- endif %}
+
 {% if s3_test | default(false) %}
+DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
+MEDIA_ROOT = ""
 AWS_ACCESS_KEY_ID = "{{ minio_access_key }}"
 AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}"
 AWS_S3_REGION_NAME = "eu-central-1"
@@ -21,6 +42,17 @@ S3_USE_SIGV4 = True
 AWS_S3_SIGNATURE_VERSION = "s3v4"
 AWS_STORAGE_BUCKET_NAME = "pulp3"
 AWS_S3_ENDPOINT_URL = "http://minio:9000"
-DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
 AWS_DEFAULT_ACL = "@none None"
 {% endif %}
+
+{% if azure_test | default(false) %}
+DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage"
+MEDIA_ROOT = ""
+AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+AZURE_ACCOUNT_NAME = "devstoreaccount1"
+AZURE_CONTAINER =  "pulp-test"
+AZURE_LOCATION = "pulp3"
+AZURE_OVERWRITE_FILES = True
+AZURE_URL_EXPIRATION_SECS = 120
+AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://ci-azurite:10000/devstoreaccount1;'
+{% endif %}

.ci/ansible/smash-config.json

@@ -5,7 +5,8 @@
       "password"
     ],
     "selinux enabled": false,
-    "version": "3"
+    "version": "3",
+    "aiohttp_fixtures_origin": "172.18.0.1"
   },
   "hosts": [
     {

.ci/ansible/start_container.yaml

@@ -12,8 +12,15 @@
         mode: "0755"
       loop:
         - settings
+        - ssh
         - ~/.config/pulp_smash
 
+    - name: Generate an OpenSSH keypair
+      community.crypto.openssh_keypair:
+        path: ssh/id_ed25519
+        type: ed25519
+      when: stream_test | default(false)
+
     - name: "Generate Pulp Settings"
       template:
         src: settings.py.j2
@@ -70,8 +77,9 @@
     - block:
         - name: "Wait for Pulp"
           uri:
-            url: "http://pulp/pulp/api/v3/status/"
-            follow_redirects: none
+            url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/"
+            follow_redirects: all
+            validate_certs: no
           register: result
           until: result.status == 200
           retries: 12

.ci/scripts/changelog.py

@@ -0,0 +1,68 @@
+import re
+import os
+import requests
+from packaging.version import Version
+from git import Repo
+
+repo = Repo(os.getcwd())
+heads = repo.git.ls_remote("--heads", "https://github.com/pulp/pulp_container.git").split("\n")
+branches = [h.split("/")[-1] for h in heads if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1])]
+branches.sort(key=lambda ver: Version(ver), reverse=True)
+
+
+def get_changelog(branch):
+    """
+    Get changelog file for a given branch.
+
+    """
+    return requests.get(
+        f"https://raw.githubusercontent.com/pulp/pulp_container/{branch}/CHANGES.rst"
+    ).text
+
+
+def get_changelog_releases(changelog):
+    """
+    Get all versions in changelog.
+
+    """
+    versions = re.findall(r"([0-9]+)\.([0-9]+)\.([0-9]+) \(", changelog)
+    return {".".join(v) for v in versions}
+
+
+def get_changelog_entry(changelog, version):
+    """
+    Get changelog entry for a given version.
+
+    """
+    entries = changelog.split(f"{version} (")[1].split("=====\n")
+    header = f"{version} ({entries[0]}=====\n"
+    text = "\n\n\n".join(entries[1].split("\n\n\n")[0:-1])
+    return header + text + "\n\n\n"
+
+
+main_changelog = get_changelog("master")
+main_entries = get_changelog_releases(main_changelog)
+entries_list = list(main_entries)
+to_add = {}
+for branch in branches:
+    changelog = get_changelog(branch)
+    entries = get_changelog_releases(changelog)
+    for entry in entries.difference(main_entries):
+        description = get_changelog_entry(changelog, entry)
+        entries_list.append(entry)
+        print(description)
+        to_add[entry] = description
+
+entries_list.sort(key=lambda ver: Version(ver), reverse=True)
+for version in sorted(to_add, key=lambda ver: Version(ver)):
+    next_version = entries_list[entries_list.index(version) + 1]
+    new_changelog = main_changelog.split(f"{next_version} (")[0] + to_add[version]
+    new_changelog = new_changelog + f"{next_version} ("
+    new_changelog = new_changelog + main_changelog.split(f"{next_version} (")[1]
+    main_changelog = new_changelog
+
+with open("CHANGES.rst", "w") as f:
+    f.write(main_changelog)
+
+if to_add:
+    repo.git.commit("-m", "Update Changelog\n\n[noissue]", "CHANGES.rst")

.ci/scripts/check_gettext.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_container' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+# make sure this script runs at the repo root
+cd "$(dirname "$(realpath -e "$0")")"/../..
+
+set -uv
+
+# check for imports not from pulpcore.plugin. exclude tests
+MATCHES=$(grep -n -r --include \*.py "_(f")
+
+if [ $? -ne 1 ]; then
+  printf "\nERROR: Detected mix of f-strings and gettext:\n"
+  echo "$MATCHES"
+  exit 1
+fi

.ci/scripts/cherrypick.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_container' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+set -e
+
+if [ ! -d CHANGES ]; then
+  echo "Error: no CHANGES directory detected. This script must be run from the project root."
+  exit 1
+fi
+
+if [ $# -lt 3 ]
+then
+  echo "Usage: .ci/scripts/cherrypick.sh [commit-hash] [original-issue-id] [backport-issue-id]"
+  echo "   ex: .ci/scripts/cherrypick.sh abcd1234 1234 4567"
+  echo ""
+  echo "Note: make sure you are on a fork of the release branch before running this script."
+  exit
+fi
+
+commit="$(git rev-parse $1)"
+issue="$2"
+backport="$3"
+commit_message=$(git log --format=%B -n 1 $commit)
+
+if ! echo $commit_message | tr '[:upper:]' '[:lower:]' | grep -q "\[noissue\]"
+then
+  if ! echo $commit_message | tr '[:upper:]' '[:lower:]' | grep -q -E "(fixes|closes).*#$issue"
+  then
+    echo "Error: issue $issue not detected in commit message." && exit 1
+  fi
+fi
+
+if [ "$4" = "--continue" ]
+then
+  echo "Continue after manually resolving conflicts..."
+elif [ "$4" = "" ]
+then
+  if ! git cherry-pick --no-commit "$commit"
+  then
+    echo "Please resolve and add merge conflicts and restart this command with appended '--continue'."
+    exit 1
+  fi
+else
+  exit 1
+fi
+
+for file in $(find CHANGES -name "$issue.*")
+do
+  newfile="${file/$issue/$backport}"
+  git mv "$file" "$newfile"
+  sed -i -e "\$a (backported from #$issue)" "$newfile"
+  git add "$newfile"
+done
+
+commit_message="$(printf "$commit_message" | sed -E 's/(fixes|closes)/backports/i')"
+commit_message="$commit_message
+
+fixes #$backport
+
+(cherry picked from commit $commit)"
+git commit -m "$commit_message"
+
+printf "\nSuccessfully backported commit $1.\n"

.ci/scripts/redmine.py

@@ -12,10 +12,15 @@
 
 REDMINE_API_KEY = os.environ["REDMINE_API_KEY"]
 REDMINE_QUERY_URL = sys.argv[1]
+MILESTONE_URL = sys.argv[2]
+RELEASE = sys.argv[3]
 CLOSED_CURRENTRELEASE = 11
 
 redmine = Redmine(REDMINE_QUERY_URL.split("issues")[0], key=REDMINE_API_KEY)
 query_issues = REDMINE_QUERY_URL.split("=")[-1].split(",")
+milestone_name = redmine.version.get(MILESTONE_URL.split("/")[-1].split(".")[0]).name
+if milestone_name != RELEASE:
+    raise RuntimeError(f"Milestone name, '{milestone_name}', does not match version, '{RELEASE}'.")
 
 to_update = []
 for issue in query_issues: