Add user defined collections

This patch allows the deployer to create a file which adds to, or
overrides the ansible collections specified in the openstack-ansible
repo ansible-collection-requirements.yml file.

The file /etc/openstack_deploy/user-collection-requirements.yml is
read during installation of ansible collections with the bootstrap_ansible.sh
script. Any extra collections listed will be installed and will take
precedence over collections of the same name specified in
ansible-collection-requirements.yml

Depends-On: https://review.opendev.org/754706
Change-Id: I22b55291c6898312a26cc5dbf7a3ae2f327429bd

scripts/bootstrap-ansible.sh

@@ -26,6 +26,7 @@ export ANSIBLE_PACKAGE=${ANSIBLE_PACKAGE:-"ansible-base==2.10.1"}
 export ANSIBLE_ROLE_FILE=${ANSIBLE_ROLE_FILE:-"ansible-role-requirements.yml"}
 export ANSIBLE_COLLECTION_FILE=${ANSIBLE_COLLECTION_FILE:-"ansible-collection-requirements.yml"}
 export USER_ROLE_FILE=${USER_ROLE_FILE:-"user-role-requirements.yml"}
+export USER_COLLECTION_FILE=${USER_COLLECTION_FILE:-"user-collection-requirements.yml"}
 export SSH_DIR=${SSH_DIR:-"/root/.ssh"}
 export DEBIAN_FRONTEND=${DEBIAN_FRONTEND:-"noninteractive"}
 # check whether to install the ARA callback plugin
@@ -149,8 +150,6 @@ pushd /opt/ansible-runtime/bin
   done
 popd
 
-/opt/ansible-runtime/bin/ansible-galaxy collection install -r ${ANSIBLE_COLLECTION_FILE} -p /etc/ansible
-
 # Write the OSA Ansible rc file
 sed "s|OSA_INVENTORY_PATH|${OSA_INVENTORY_PATH}|g" scripts/openstack-ansible.rc > /usr/local/bin/openstack-ansible.rc
 sed -i "s|OSA_PLAYBOOK_PATH|${OSA_PLAYBOOK_PATH}|g" /usr/local/bin/openstack-ansible.rc
@@ -189,6 +188,9 @@ if [ -f "${ANSIBLE_ROLE_FILE}" ] && [[ -z "${SKIP_OSA_ROLE_CLONE+defined}" ]]; t
     export ANSIBLE_COLLECTIONS_PATH="/etc/ansible"
 
     pushd scripts
+      /opt/ansible-runtime/bin/ansible-playbook get-ansible-collection-requirements.yml \
+                       -e collection_file="${ANSIBLE_COLLECTION_FILE}" -e user_collection_file="${USER_COLLECTION_FILE}"
+
       /opt/ansible-runtime/bin/ansible-playbook get-ansible-role-requirements.yml \
                        -e role_file="${ANSIBLE_ROLE_FILE}" -e user_role_file="${USER_ROLE_FILE}"
     popd

scripts/get-ansible-collection-requirements.yml

@@ -0,0 +1,77 @@
+---
+# Copyright 2020 BBC R&D.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Clone the role ansible-role-requirements
+  hosts: localhost
+  connection: local
+  user: root
+  gather_facts: false
+  tasks:
+    - name: Ensure the default collections directory exists
+      file:
+        path: "{{ collection_path_default }}"
+        state: directory
+
+    - name: Read the list of user collections
+      set_fact:
+        user_collection_names: "{{ user_collections.collections | default([]) | map(attribute='name') | list }}"
+
+    - name: Generate a list of required collections excluding user overridden collections
+      set_fact:
+         galaxy_collections_list : "{{ galaxy_collections_list + [ item ] }}"
+      when:
+        - item.name not in user_collection_names
+      with_items: "{{ required_collections.collections }}"
+
+    - name: Append user collections to filtered required collections
+      set_fact:
+        galaxy_collections_list: "{{ galaxy_collections_list + [ item ] }}"
+      with_items: "{{ user_collections.collections }}"
+      when:
+        - user_collections.collections is defined
+
+    - name: Create temporary file for galaxy collection requirements
+      tempfile:
+      register: collection_requirements_tmpfile
+
+    - name: Copy content into galaxy collection requirements temporary file
+      copy:
+        content: "{{ galaxy_collections | to_nice_yaml }}"
+        dest: "{{ collection_requirements_tmpfile.path }}"
+
+    - name: Install collection requirements with ansible galaxy
+      command: >
+        /opt/ansible-runtime/bin/ansible-galaxy collection install
+        -r "{{ collection_requirements_tmpfile.path }}"
+        -p "{{ collection_path_default }}"
+      register: collection_install
+
+    - name: Show collection install output
+      debug: msg="{{ collection_install.stdout.split('\n') }}"
+
+    - name: Clean up temporary file
+      file:
+        path: "{{ collection_requirements_tmpfile.path }}"
+        state: absent
+
+  vars:
+    galaxy_collections_list: []
+    galaxy_collections:
+      collections: "{{ galaxy_collections_list }}"
+    collections_file: "{{ playbook_dir }}/../ansible-collection-requirements.yml"
+    required_collections: "{{ lookup('file', collections_file) | from_yaml }}"
+    collection_path_default: '/etc/ansible/'
+    user_collections: "{{ lookup('file', user_collections_path, errors='ignore')|default([], true) | from_yaml }}"
+    user_collections_path: "{{ lookup('env', 'OSA_CONFIG_DIR') | default('/etc/openstack_deploy', true) ~ '/' ~ (user_collection_file|default('')) }}"