containers: Build both stable and dev images

and make images far more configurable.

Implementation includes:

Build stable and master (latest) versions (tags) of both images.

Use an undefined plugins list for no plugins ather than the
dummy pulpcore-plugin.

Use variables for the pulpcore & pulpcore-plugin pip installs

Use vars/vars.yml as an override for vars
(including the images data structure.)

Start work on installing into the container, and limit the CI builds
to only pulpcore & pulpcore-plugin stable.

fixes: #5116
Handle properly whether to build container images with pulpcore stable or dev
(Actually does more than that.)
https://pulp.plan.io/issues/5116

Includes some work on:
re: #5062
Create pulpcore and pulp_file container images automatically via CI
https://pulp.plan.io/issues/5062

.gitignore

@@ -44,3 +44,6 @@ Vagrantfile
 docs/_build
 docs/_diagrams
 docs/api.yaml
+
+# Overriden variables
+containers/vars/vars.yaml

.travis/post_script.sh

@@ -2,4 +2,32 @@
 
 cd containers
 
+# FIXME: This won't work  because not only is it outside the Docker build env,
+# it's outside of the build context (the pwd from which `docker build` is run.)
+# We need to find/develop a Dockerfile COPY solution for this.
+: '
+if [ -n "$PULP_PLUGIN_PR_NUMBER" ]; then
+  PULPCORE_PLUGIN_PATH=../../pulpcore-plugin
+else
+  PULPCORE_PLUGIN_PATH=git+https://github.com/pulp/pulpcore-plugin.git
+fi
+cat > vars/vars.yaml << VARSYAML
+---
+images:
+  - pulpcore_$(git rev-parse --abbrev-ref HEAD | tr '-' '_'):
+      image_name: pulpcore
+      tag: $(git rev-parse --abbrev-ref HEAD)
+      pulpcore: ..
+      pulpcore_plugin: $PULPCORE_PLUGIN_PATH
+VARSYAML
+'
+
+cat > vars/vars.yaml << VARSYAML
+---
+images:
+  - pulpcore_$(git rev-parse --abbrev-ref HEAD | tr '-' '_'):
+      image_name: pulpcore
+      tag: $(git rev-parse --abbrev-ref HEAD)
+VARSYAML
+
 ansible-playbook build.yaml

containers/README.md

@@ -1,6 +1,10 @@
 # Pulp 3 Containers
 
-This directory contains assets and tooling for building Pulp 3 container images. The images differ based on the set of plugins installed, including none (pulpcore-plugin only.) Each image is an all-in-one image that gets called with different commands to assume each of the roles. The roles must be specified as the container's "command", without the full filepath:
+This directory contains assets and tooling for building Pulp 3 container images.
+
+The image names differ based on the set of plugins installed, including none (pulpcore-plugin only). The image tags differ based on the versions of pulpcore & the plugins.
+
+Each image is an all-in-one image that gets called with different commands to assume each of the roles. The roles must be specified as the container's "command", without the full filepath:
 
 * pulp-api
 * pulp-content
@@ -15,22 +19,16 @@ The images can be built with the help of an Ansible playbook. To build the image
 
     ansible-playbook build.yaml
 
-The variable "images" can be overriden. It includes specifying the plugins per image.
-
-The default value (yaml converted to json):
-
-    ansible-playbook build.yaml -e '{"images":[{"pulpcore":{"plugins":["pulpcore-plugin"]}},{"pulp":{"plugins":["pulp-file","pulp-ansible","pulp-cookbook","pulp-docker","pulp-maven","pulp-python","pulp-rpm"]}}]}'
-
-The default value, but with nightly versions of plugins instead:
-
-    ansible-playbook build.yaml -e '{"images":[{"pulpcore":{"plugins":["git+https://github.com/pulp/pulpcore-plugin.git"]}},{"pulp":{"plugins":["git+https://github.com/pulp/pulp_file.git","git+https://github.com/pulp/pulp_ansible.git","git+https://github.com/gmbnomis/pulp_cookbook.git","git+https://github.com/pulp/pulp_docker.git","git+https://github.com/pulp/pulp_maven.git","git+https://github.com/pulp/pulp_python.git","git+https://github.com/pulp/pulp_rpm.git"]}}]}'
+See `vars/defaults.yaml` for how to customize the "images" variable (data structure.)
 
 ## Push Image to Registry
 
-The built image can be pushed to a registry using an Ansible playbook. The default configuration will attempt to push the image to `quay.io/pulp`:
+The built image can be pushed to a registry using an Ansible playbook. The default configuration will attempt to push the images to `quay.io/pulp`:
 
     ansible-playbook push.yaml
 
 The image can be pushed to custom registry by specifying variables via the command line:
 
     ansible-playbook push.yaml -e registry=docker.io -e project=myproject
+
+See `vars/defaults.yaml` for more info about the variables.

containers/build.yaml

@@ -6,21 +6,14 @@
   vars_files:
     - vars/defaults.yaml
   tasks:
-    - name: 'Check for podman'
-      command: 'which podman'
-      register: podman_exists
-      ignore_errors: true
-
-    - set_fact:
-        container_cli: 'podman'
-      when: podman_exists | success
+    - import_tasks: common_tasks.yaml
 
     - name: 'Build images'
-      command: "{{ container_cli }} build --network host --no-cache --build-arg VERSION={{ tag }} --build-arg PLUGINS=\"{{ item.value.plugins | join(' ') }}\" -t {{ item.key }}:{{ tag }} ."
+      command: "{{ container_cli }} build --network host --no-cache --build-arg VERSION={{ item.value.tag }} --build-arg PULPCORE={{ item.value.pulpcore | default('pulpcore') }} --build-arg PULPCORE_PLUGIN={{ item.value.pulpcore_plugin | default('pulpcore-plugin') }} --build-arg PLUGINS=\"{{ item.value.plugins | default([]) | join(' ') }}\" -t {{ item.value.image_name }}:{{ item.value.tag }} ."
       args:
         chdir: images/pulp
       with_dict: "{{ images }}"
 
     - name: 'Tag images'
-      command: "{{ container_cli }} tag {{ item.key }}:{{ tag }} {{ registry }}/{{ project }}/{{ item.key }}:{{ tag }}"
+      command: "{{ container_cli }} tag {{ item.value.image_name }}:{{ item.value.tag }} {{ registry }}/{{ project }}/{{ item.value.image_name }}:{{ item.value.tag }}"
       with_dict: "{{ images }}"

containers/common_tasks.yaml

@@ -0,0 +1,19 @@
+---
+- name: 'Check for podman'
+  command: 'which podman'
+  register: podman_exists
+  ignore_errors: true
+
+- set_fact:
+    container_cli: 'podman'
+  when: podman_exists | success
+
+- name: Check if vars.yaml was created by the user
+  stat:
+    path: vars/vars.yaml
+  register: varsyaml
+
+- name: include vars.yaml
+  include_vars:
+    vars.yaml
+  when: varsyaml.stat.exists

containers/images/pulp/Dockerfile

@@ -1,6 +1,8 @@
 FROM fedora:30
 
-ARG PLUGINS=""
+ARG PULPCORE
+ARG PULPCORE_PLUGIN
+ARG PLUGINS
 
 # https://superuser.com/questions/959380/how-do-i-install-generate-all-locales-on-fedora
 # This may not be necessary anymore because Fedora 30, unlike CentOS 7, has
@@ -50,10 +52,10 @@ RUN ln -s /usr/bin/pip3 /usr/local/bin/pip
 RUN mkdir -p /etc/pulp
 
 RUN pip install gunicorn
-RUN pip install git+https://github.com/pulp/pulpcore.git
-RUN pip install git+https://github.com/pulp/pulpcore-plugin.git
-RUN pip install git+https://github.com/pulp/pulpcore.git#egg=pulpcore[postgres]
-RUN pip install $PLUGINS
+RUN pip install $PULPCORE
+RUN pip install $PULPCORE_PLUGIN
+RUN pip install ${PULPCORE}[postgres]
+RUN test -z "$PLUGINS" || pip install $PLUGINS
 
 RUN mkdir -p /usr/local/lib/python3.7/site-packages/pulpcore/app/migrations
 RUN mkdir -p /usr/local/lib/python3.7/site-packages/pulp_file/app/migrations

containers/push.yaml

@@ -5,19 +5,12 @@
   vars_files:
     - vars/defaults.yaml
   tasks:
-    - name: 'Check for podman'
-      command: 'which podman'
-      register: podman_exists
-      ignore_errors: true
-
-    - set_fact:
-        container_cli: 'podman'
-      when: podman_exists | success
+    - import_tasks: common_tasks.yaml
 
     - name: 'Tag images'
-      command: "{{ container_cli }} tag {{ item.key }}:{{ tag }} {{ registry }}/{{ project }}/{{ item.key }}:{{ tag }}"
+      command: "{{ container_cli }} tag {{ item.value.image_name }}:{{ item.value.tag }} {{ registry }}/{{ project }}/{{ item.value.image_name }}:{{ item.value.tag }}"
       with_dict: "{{ images }}"
 
     - name: 'Push images'
-      command: "{{ container_cli }} push {{ registry }}/{{ project }}/{{ item.key }}:{{ tag }}"
+      command: "{{ container_cli }} push {{ registry }}/{{ project }}/{{ item.value.image_name }}:{{ item.value.tag }}"
       with_dict: "{{ images }}"

containers/vars/defaults.yaml

@@ -1,9 +1,47 @@
 ---
+# Default variables for building the container images.
+#
+# To override, either:
+# a. Create vars.yaml in this dir
+# b: Pass JSON args to ansible's -e argument.
+#    PROTIP: (YAML to JSON converters exist on the web.)
+# Either way, each variable can be omitted (left default) or overriden, but not
+# have a null value.
+# However, the entire images data structure is one variable. It must be
+# entirely overriden or entirely defaulted to.
+#
+# For each image in images:
+# key: Must be unique, but any name you'd like. Outputted during Ansible tasks.
+# image_name: The name ("repo") of the image. Mandatory.
+# tag: The tag (version string) for the image. Mandatory.
+# pulpcore: The pip install string for pulpcore. Must accept [postgres] being
+#           appended to the end, so for git URLs, keep #egg=pulpcore on the end.
+#           Defaults to "pulpcore" (latest stable from PyPi) via playbook logic.
+#           To keep the default, leave undefined, do not let it be a null value.
+# pulpcore-plugin: The pip install string for pulpcore-plugin.
+#                  Defaults to "pulpcore-plugin" (latest stable from PyPi) via
+#                  playbook logic. To keep the default, leave undefined, do not
+#                  let it be a null value.
+# plugins: List of pip install strings for plugins.
+#          To specify no plugins, leave undefined, do not let it be a null value.
+#
+# registry: The image registry to push to, and to tag for.
+# project: The project (username/organization) on the registry to push to,
+#          and to tag for.
 images:
-  - pulpcore:
-      plugins:
-        - pulpcore-plugin
-  - pulp:
+  - pulpcore_stable:
+      image_name: pulpcore
+      tag: stable
+  - pulpcore_master:
+      image_name: pulpcore
+      tag: latest
+      # Must specify egg name here so that the Dockerfile can specify the
+      # setuptools extra of postgres
+      pulpcore: git+https://github.com/pulp/pulpcore.git#egg=pulpcore
+      pulpcore_plugin: git+https://github.com/pulp/pulpcore-plugin.git
+  - pulp_stable_plugins_stable:
+      image_name: pulp
+      tag: stable
       plugins:
         - pulp-file
         - pulp-ansible
@@ -12,6 +50,18 @@ images:
         - pulp-maven
         - pulp-python
         - pulp-rpm
+  - pulp_master_plugins_master:
+      image_name: pulp
+      tag: latest
+      pulpcore: git+https://github.com/pulp/pulpcore.git#egg=pulpcore
+      pulpcore_plugin: git+https://github.com/pulp/pulpcore-plugin.git
+      plugins:
+        - "git+https://github.com/pulp/pulp_file.git"
+        - "git+https://github.com/pulp/pulp_ansible.git"
+        - "git+https://github.com/gmbnomis/pulp_cookbook.git"
+        - "git+https://github.com/pulp/pulp_docker.git"
+        - "git+https://github.com/pulp/pulp_maven.git"
+        - "git+https://github.com/pulp/pulp_python.git"
+        - "git+https://github.com/pulp/pulp_rpm.git"
 registry: quay.io
 project: pulp
-tag: latest