diff --git a/.travis/Containerfile.j2 b/.travis/Containerfile.j2
index 7f8a8def..5bc4188d 100644
--- a/.travis/Containerfile.j2
+++ b/.travis/Containerfile.j2
@@ -1,4 +1,4 @@
-FROM pulp/pulp-ci:latest
+FROM {{ ci_base | default("pulp/pulp-ci:latest") }}
 
 {% if s3_test | default(false) %}
 # Hacking botocore (https://github.com/boto/botocore/pull/1990)
diff --git a/.travis/before_install.sh b/.travis/before_install.sh
index 9fcadd40..0ba2030b 100755
--- a/.travis/before_install.sh
+++ b/.travis/before_install.sh
@@ -9,6 +9,9 @@
 
 set -mveuo pipefail
 
+mkdir .travis/vars || true
+echo "---" > .travis/vars/main.yaml
+
 export PRE_BEFORE_INSTALL=$TRAVIS_BUILD_DIR/.travis/pre_before_install.sh
 export POST_BEFORE_INSTALL=$TRAVIS_BUILD_DIR/.travis/post_before_install.sh
 
@@ -29,10 +32,12 @@ then
   export PULPCORE_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}')
   export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}')
   export PULP_OPENAPI_GENERATOR_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}')
+  echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .travis/vars/main.yaml
 else
   export PULPCORE_PR_NUMBER=
   export PULP_SMASH_PR_NUMBER=
   export PULP_OPENAPI_GENERATOR_PR_NUMBER=
+  export CI_BASE_IMAGE=
 fi
 
 # dev_requirements contains tools needed for flake8, etc.
diff --git a/.travis/before_script.sh b/.travis/before_script.sh
index d448c788..43c4b28c 100755
--- a/.travis/before_script.sh
+++ b/.travis/before_script.sh
@@ -18,6 +18,10 @@ if [[ -f $PRE_BEFORE_SCRIPT ]]; then
   source $PRE_BEFORE_SCRIPT
 fi
 
+# Developers should be able to reproduce the containers with this config
+echo "CI vars:"
+tail -v -n +1 .travis/vars/main.yaml
+
 # Developers often want to know the final pulp config
 echo "PULP CONFIG:"
 tail -v -n +1 .travis/settings/settings.* ~/.config/pulp_smash/settings.json
diff --git a/.travis/build_container.yaml b/.travis/build_container.yaml
index 0e39f62c..5392e73d 100644
--- a/.travis/build_container.yaml
+++ b/.travis/build_container.yaml
@@ -11,7 +11,6 @@
         dest: Containerfile
 
     - name: "Build pulp image"
-      docker_image:
       # We build from the ../.. (parent dir of pulpcore git repo) Docker build
       # "context" so that repos like pulp-smash are accessible to Docker
       # build. So that PR branches can be used via relative paths.
@@ -20,15 +19,7 @@
       # 1-off-builds and Travis CI purposes (which has no cache across CI runs.)
       # Run build.yaml with -e cache=false if your builds are using outdated
       # layers.
-        name: "{{ image.name }}"
-        tag: "{{ image.tag }}"
-        build:
-          path: "../.."
-          dockerfile: "{{ playbook_dir }}/Containerfile"
-          nocache: "{{ not cache | default(true) | bool }}"
-          pull: false
-        state: present
-        source: build
+      command: "docker build --network host --no-cache={{ not cache | default(true) | bool }} -t {{ image.name }}:{{ image.tag }} -f {{ playbook_dir }}/Containerfile ../.."
 
     - name: "Clean image cache"
       docker_prune:
diff --git a/.travis/install.sh b/.travis/install.sh
index 2b80e972..d6308ee8 100755
--- a/.travis/install.sh
+++ b/.travis/install.sh
@@ -44,13 +44,10 @@ else
   # Fallback
   TAG=$(git rev-parse --abbrev-ref HEAD | tr / _)
 fi
-
-mkdir vars
 if [ -n "$TRAVIS_TAG" ]; then
   # Install the plugin only and use published PyPI packages for the rest
   # Quoting ${TAG} ensures Ansible casts the tag as a string.
-  cat > vars/main.yaml << VARSYAML
----
+  cat >> vars/main.yaml << VARSYAML
 image:
   name: pulp
   tag: "${TAG}"
@@ -66,8 +63,7 @@ services:
       - ./settings:/etc/pulp
 VARSYAML
 else
-  cat > vars/main.yaml << VARSYAML
----
+  cat >> vars/main.yaml << VARSYAML
 image:
   name: pulp
   tag: "${TAG}"
diff --git a/.travis/redmine.py b/.travis/redmine.py
new file mode 100644
index 00000000..742b0db7
--- /dev/null
+++ b/.travis/redmine.py
@@ -0,0 +1,30 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --travis pulp_python' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+import os
+import sys
+
+from redminelib import Redmine
+
+REDMINE_API_KEY = os.environ["REDMINE_API_KEY"]
+REDMINE_QUERY_URL = sys.argv[1]
+CLOSED_CURRENTRELEASE = 11
+
+redmine = Redmine(REDMINE_QUERY_URL.split("issues")[0], key=REDMINE_API_KEY)
+query_issues = REDMINE_QUERY_URL.split("=")[-1].split(",")
+
+to_update = []
+for issue in query_issues:
+    status = redmine.issue.get(int(issue)).status.name
+    if "CLOSE" not in status and status != "MODIFIED":
+        raise ValueError("One or more issues are not MODIFIED")
+    if status == "MODIFIED":  # Removing the already closed
+        to_update.append(int(issue))
+
+for issue in to_update:
+    print(f"Closing #{issue}")
+    redmine.issue.update(issue, status_id=CLOSED_CURRENTRELEASE)
diff --git a/.travis/release.py b/.travis/release.py
index fe6a95c6..2897f428 100644
--- a/.travis/release.py
+++ b/.travis/release.py
@@ -1,27 +1,81 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --travis pulp_python' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
 import argparse
+import json
 import os
 import textwrap
+from collections import defaultdict
+from pathlib import Path
 
 from git import Repo
+from redminelib import Redmine
+
+
+REDMINE_URL = "https://pulp.plan.io"
+REDMINE_QUERY_URL = f"{REDMINE_URL}/issues?set_filter=1&status_id=*&issue_id="
+
+
+def validate_redmine_data(redmine_query_url, redmine_issues):
+    """Validate redmine milestone."""
+    redmine = Redmine("https://pulp.plan.io")
+    project_set = set()
+    stats = defaultdict(list)
+    milestone_url = "\n[noissue]"
+    for issue in redmine_issues:
+        redmine_issue = redmine.issue.get(int(issue))
+
+        project_name = redmine_issue.project.name
+        project_set.update([project_name])
+        stats[f"project_{project_name.lower().replace(' ', '_')}"].append(issue)
+
+        status = redmine_issue.status.name
+        if "CLOSE" not in status and status != "MODIFIED":
+            stats["status_not_modified"].append(issue)
+
+    print(f"\n\nRedmine stats: {json.dumps(stats, indent=2)}")
+    error_messages = []
+    if stats.get("status_not_modified"):
+        error_messages.append(f"One or more issues are not MODIFIED {stats['status_not_modified']}")
+    if stats.get("without_milestone"):
+        error_messages.append(
+            f"One or more issues are not associated with a milestone {stats['without_milestone']}"
+        )
+    if len(project_set) > 1:
+        error_messages.append(f"Issues with different projects - {project_set}")
+    if error_messages:
+        error_messages.append(f"Verify at {redmine_query_url}")
+        raise RuntimeError("\n".join(error_messages))
+
+    return milestone_url
 
 
-REDMINE_QUERY_URL = "https://pulp.plan.io/issues?set_filter=1&status_id=*&issue_id="
 release_path = os.path.dirname(os.path.abspath(__file__))
 plugin_path = release_path
 if ".travis" in release_path:
     plugin_path = os.path.dirname(release_path)
 
 version = {}
-with open(f"{plugin_path}/pulp_python/__init__.py") as fp:
+plugin_name = "pulp_python"
+with open(f"{plugin_path}/{plugin_name}/__init__.py") as fp:
     version_line = [line for line in fp.readlines() if "__version__" in line][0]
     exec(version_line, version)
 release_version = version["__version__"].replace(".dev", "")
 
-to_close = []
-for filename in os.listdir(f"{plugin_path}/CHANGES"):
-    if filename.split(".")[0].isdigit():
-        to_close.append(filename.split(".")[0])
-issues = ",".join(to_close)
+issues_to_close = []
+for filename in Path(f"{plugin_path}/CHANGES").rglob("*"):
+    if filename.stem.isdigit():
+        issue = filename.stem
+        issue_url = f"{REDMINE_URL}/issues/{issue}.json"
+        issues_to_close.append(issue)
+
+issues = ",".join(issues_to_close)
+redmine_final_query = f"{REDMINE_QUERY_URL}{issues}"
+milestone_url = validate_redmine_data(redmine_final_query, issues_to_close)
 
 helper = textwrap.dedent(
     """\
@@ -89,12 +143,13 @@
 
 os.system("bump2version release --allow-dirty")
 
-plugin_name = plugin_path.split("/")[-1]
 git.add(f"{plugin_path}/{plugin_name}/__init__.py")
 git.add(f"{plugin_path}/setup.py")
 git.add(f"{plugin_path}/requirements.txt")
 git.add(f"{plugin_path}/.bumpversion.cfg")
-git.commit("-m", f"Releasing {release_version}\n\n[noissue]")
+git.commit(
+    "-m", f"Releasing {release_version}\n\nRedmineQuery: {redmine_final_query}\n{milestone_url}"
+)
 
 sha = repo.head.object.hexsha
 short_sha = git.rev_parse(sha, short=7)
@@ -102,7 +157,7 @@
 # Third commit: bump to .dev
 with open(f"{plugin_path}/requirements.txt", "wt") as setup_file:
     for line in setup_lines:
-        if "pulpcore" in line and "pulpcore" not in release_path:
+        if "pulpcore" in line and "pulpcore" not in release_path and release_type != "patch":
             line = f"pulpcore>={lower_pulpcore_version}\n"
 
         setup_file.write(line)
@@ -110,7 +165,7 @@
 os.system(f"bump2version {release_type} --allow-dirty")
 
 version = {}
-with open(f"{plugin_path}/pulp_python/__init__.py") as fp:
+with open(f"{plugin_path}/{plugin_name}/__init__.py") as fp:
     version_line = [line for line in fp.readlines() if "__version__" in line][0]
     exec(version_line, version)
 new_dev_version = version["__version__"]
@@ -122,6 +177,6 @@
 git.add(f"{plugin_path}/.bumpversion.cfg")
 git.commit("-m", f"Bump to {new_dev_version}\n\n[noissue]")
 
-print(f"\n\nRedmine query of issues to close:\n{REDMINE_QUERY_URL}{issues}")
+print(f"\n\nRedmine query of issues to close:\n{redmine_final_query}")
 print(f"Release commit == {short_sha}")
 print(f"All changes were committed on branch: release_{release_version}")
diff --git a/.travis/release_requirements.txt b/.travis/release_requirements.txt
new file mode 100644
index 00000000..a2254f00
--- /dev/null
+++ b/.travis/release_requirements.txt
@@ -0,0 +1,3 @@
+bump2version
+gitpython
+python-redmine
diff --git a/.travis/settings.py.j2 b/.travis/settings.py.j2
index e934727c..56794bc0 100644
--- a/.travis/settings.py.j2
+++ b/.travis/settings.py.j2
@@ -1,7 +1,10 @@
 CONTENT_ORIGIN = "http://pulp:80"
 ANSIBLE_API_HOSTNAME = "http://pulp:80"
 ANSIBLE_CONTENT_HOSTNAME = "http://pulp:80/pulp/content"
-TOKEN_AUTH_DISABLED = True
+PRIVATE_KEY_PATH = "/etc/pulp/certs/token_private_key.pem"
+PUBLIC_KEY_PATH = "/etc/pulp/certs/token_public_key.pem"
+TOKEN_SERVER = "http://pulp:80/token"
+TOKEN_SIGNATURE_ALGORITHM = "ES256"
 
 {% if pulp_settings %}
 {% for key, value in pulp_settings.items() %}
diff --git a/.travis/start_container.yaml b/.travis/start_container.yaml
index d0394846..9217d24e 100644
--- a/.travis/start_container.yaml
+++ b/.travis/start_container.yaml
@@ -67,14 +67,27 @@
         state: present
       when: s3_test | default(false)
 
-    - name: "Wait for Pulp"
-      uri:
-        url: "http://pulp/pulp/api/v3/status/"
-        follow_redirects: none
-      register: result
-      until: result.status == 200
-      retries: 6
-      delay: 5
+    - block:
+        - name: "Wait for Pulp"
+          uri:
+            url: "http://pulp/pulp/api/v3/status/"
+            follow_redirects: none
+          register: result
+          until: result.status == 200
+          retries: 6
+          delay: 5
+      rescue:
+        - name: "Output pulp container log"
+          command: "docker logs pulp"
+          failed_when: true
+
+    - name: "Set pulp password in .netrc"
+      copy:
+        dest: "~/.netrc"
+        content: |
+          machine pulp
+          login admin
+          password password
 
 - hosts: pulp
   gather_facts: false
diff --git a/.travis/update_redmine.sh b/.travis/update_redmine.sh
new file mode 100644
index 00000000..6d583ffa
--- /dev/null
+++ b/.travis/update_redmine.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -euv
+
+export COMMIT_MSG=$(git log --format=%B --no-merges -1)
+export RELEASE=$(echo $COMMIT_MSG | awk '{print $2}')
+export MILESTONE_URL=$(echo $COMMIT_MSG | awk '{print $6}')
+export REDMINE_QUERY_URL=$(echo $COMMIT_MSG | awk '{print $4}')
+
+echo "Releasing $RELEASE"
+echo "Milestone URL: $MILESTONE_URL"
+echo "Query: $REDMINE_QUERY_URL"
+
+MILESTONE=$(http $MILESTONE_URL | jq -r .version.name)
+echo "Milestone: $MILESTONE"
+
+if [[ "$MILESTONE" != "$RELEASE" ]]; then
+  echo "Milestone $MILESTONE is not equal to Release $RELEASE"
+  exit 1
+fi
+
+pip install python-redmine
+python .travis/redmine.py $REDMINE_QUERY_URL
diff --git a/.travis/validate_commit_message.py b/.travis/validate_commit_message.py
index 236a655d..49aa6c98 100644
--- a/.travis/validate_commit_message.py
+++ b/.travis/validate_commit_message.py
@@ -5,12 +5,11 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
-import glob
-import os
 import re
 import requests
 import subprocess
 import sys
+from pathlib import Path
 
 KEYWORDS = ["fixes", "closes", "re", "ref"]
 NO_ISSUE = "[noissue]"
@@ -42,12 +41,12 @@ def __check_status(issue):
 
 
 def __check_changelog(issue):
-    matches = glob.glob(f"CHANGES/**/{issue}.*", recursive=True)
+    matches = list(Path("CHANGES").rglob(f"{issue}.*"))
 
     if len(matches) < 1:
         sys.exit(f"Could not find changelog entry in CHANGES/ for {issue}.")
     for match in matches:
-        if os.path.splitext(match)[1] not in CHANGELOG_EXTS:
+        if match.suffix not in CHANGELOG_EXTS:
             sys.exit(f"Invalid extension for changelog entry '{match}'.")
 
 
diff --git a/template_config.yml b/template_config.yml
index 5de5599d..3a037b39 100644
--- a/template_config.yml
+++ b/template_config.yml
@@ -34,4 +34,5 @@ test_performance: false
 test_s3: false
 travis_addtl_services: []
 travis_notifications: None
+update_redmine: false