Merge upstream stein

docker/aodh/aodh-api/extend_start.sh

@@ -5,6 +5,7 @@
 if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
     # Loading Apache2 ENV variables
     . /etc/apache2/envvars
+    install -d /var/run/apache2/
     rm -rf /var/run/apache2/*
 else
     rm -rf /var/run/httpd/* /run/httpd/* /tmp/httpd*

docker/barbican/barbican-api/extend_start.sh

@@ -14,6 +14,7 @@ if [[ "$(whoami)" == 'root' ]]; then
     if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
         # Loading Apache2 ENV variables
         . /etc/apache2/envvars
+        install -d /var/run/apache2/
         rm -rf /var/run/apache2/*
     else
         rm -rf /var/run/httpd/* /run/httpd/* /tmp/httpd*

docker/base/Dockerfile.j2

@@ -295,10 +295,12 @@ COPY sources.list.{{ base_distro }} /etc/apt/sources.list
 {% else %}
 COPY sources.list.{{ base_distro }}.{{ base_arch }} /etc/apt/sources.list
 {% endif %}
+COPY sources.list /etc/apt/sources.list.d/kolla-custom.list
 {% endblock %}
 
 {% block base_ubuntu_package_apt_preferences %}
 COPY apt_preferences.{{ base_distro }} /etc/apt/preferences
+COPY apt_preferences /etc/apt/preferences.d/kolla-custom
 {% endblock %}
 
 {% set base_apt_packages = [

docker/base/apt_preferences

@@ -0,0 +1 @@
+# used for adding custom options by 'apt_preferences' option of kolla-build.conf

docker/base/apt_preferences.debian

@@ -0,0 +1,29 @@
+# We do not want packages from this repo
+Package: *
+Pin: release o=obs://private/home:marcin.juszkiewicz/debian-buster
+Pin-Priority: 100
+
+# Unless it is kibana (on aarch64)
+Package: kibana
+Pin: release o=obs://private/home:marcin.juszkiewicz/debian-buster
+Pin-Priority: 600
+
+# on x86-64 use upstream package for kibana
+Package: kibana
+Pin: release o=elastic,b=amd64
+Pin-Priority: 700
+
+# We need fixed libvirt for aarch64
+Package: *libvirt*
+Pin: release o=obs://private/home:marcin.juszkiewicz/debian-buster
+Pin-Priority: 600
+
+# on x86-64 we want libvirt from Debian
+Package: *libvirt*
+Pin: release o=Debian,b=amd64
+Pin-Priority: 700
+
+# on ppc64le we want libvirt from Debian
+Package: *libvirt*
+Pin: release o=Debian,b=ppc64le
+Pin-Priority: 700

docker/base/sources.list

@@ -0,0 +1 @@
+# used for adding custom repos by 'apt_sources_list' option of kolla-build.conf

docker/base/sources.list.debian

@@ -18,3 +18,6 @@ deb http://buster-stein.debian.net/debian buster-stein-backports-nochange main
 
 # td-agent for fluentd
 deb http://packages.treasuredata.com/3/debian/stretch stretch contrib
+
+# kibana for non-x86 and fixed libvirt for aarch64
+deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./

docker/cinder/cinder-api/extend_start.sh

@@ -9,7 +9,7 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
 fi
 
 if [[ "${!KOLLA_OSM[@]}" ]]; then
-    if [[ -z ${MAX_NUMBER} ]]; then
+    if [[ "${!MAX_NUMBER[@]}" ]]; then
         cinder-manage db online_data_migrations --max_count ${MAX_NUMBER}
     else
         cinder-manage db online_data_migrations
@@ -24,6 +24,7 @@ if [[ "$(whoami)" == 'root' ]]; then
     if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
         # Loading Apache2 ENV variables
         . /etc/apache2/envvars
+        install -d /var/run/apache2/
         rm -rf /var/run/apache2/*
     else
         rm -rf /var/run/httpd/* /run/httpd/* /tmp/httpd*

docker/cloudkitty/cloudkitty-api/extend_start.sh

@@ -5,6 +5,7 @@
 if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
     # Loading Apache2 ENV variables
     . /etc/apache2/envvars
+    install -d /var/run/apache2/
     rm -rf /var/run/apache2/*
 else
     rm -rf /var/run/httpd/* /run/httpd/* /tmp/httpd*

docker/crane/extend_start.sh

@@ -7,6 +7,7 @@ if [[ "$(whoami)" == 'root' ]]; then
     if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
         # Loading Apache2 ENV variables
         . /etc/apache2/envvars
+        install -d /var/run/apache2/
         rm -rf /var/run/apache2/*
     else
         rm -rf /var/run/httpd/* /run/httpd/* /tmp/httpd*

docker/cyborg/cyborg-api/extend_start.sh

@@ -10,6 +10,7 @@ fi
 if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
     # Loading Apache2 ENV variables
     . /etc/apache2/envvars
+    install -d /var/run/apache2/
     rm -rf /var/run/apache2/*
 else
     rm -rf /var/run/httpd/* /run/httpd/* /tmp/httpd*

docker/freezer/freezer-api/extend_start.sh

@@ -10,6 +10,7 @@ fi
 if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
     # Loading Apache2 ENV variables
     . /etc/apache2/envvars
+    install -d /var/run/apache2/
     rm -rf /var/run/apache2/*
 else
     rm -rf /var/run/httpd/* /run/httpd/* /tmp/httpd*

docker/glance/glance-api/Dockerfile.j2

@@ -6,11 +6,14 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 {% import "macros.j2" as macros with context %}
 
 {% if base_distro in ['centos', 'oraclelinux'] %}
-    {% set glance_api_packages = ['qemu-img-ev'] %}
+    {% set glance_api_packages = ['nfs-utils', 'qemu-img-ev'] %}
 {% elif base_distro == 'rhel' %}
-    {% set glance_api_packages = ['qemu-img'] %}
+    {% set glance_api_packages = ['nfs-utils', 'qemu-img'] %}
 {% elif base_package_type == 'deb' %}
-    {% set glance_api_packages = ['qemu-utils'] %}
+    {% set glance_api_packages = [
+        'nfs-common',
+        'qemu-utils'
+    ] %}
 {% endif %}
 
 {{ macros.install_packages(glance_api_packages | customizable("packages")) }}

docker/gnocchi/gnocchi-api/extend_start.sh

@@ -5,6 +5,7 @@
 if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
     # Loading Apache2 ENV variables
     . /etc/apache2/envvars
+    install -d /var/run/apache2/
     rm -rf /var/run/apache2/*
 else
     rm -rf /var/run/httpd/* /run/httpd/* /tmp/httpd*

docker/heat/heat-base/extend_start.sh

@@ -14,6 +14,7 @@ if [[ "$(whoami)" == 'root' ]]; then
     if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
         # Loading Apache2 ENV variables
         . /etc/apache2/envvars
+        install -d /var/run/apache2/
         rm -rf /var/run/apache2/*
     else
         rm -rf /var/run/httpd/* /run/httpd/* /tmp/httpd*

docker/horizon/Dockerfile.j2

@@ -143,7 +143,9 @@ ADD plugins-archive /
     '/plugins/*'
 ] %}
 
+# NOTE(hrw): to install horizon from unpacked sources we cannot have it in upper-constraints.txt
 RUN ln -s horizon-source/* horizon \
+    && sed -i /^horizon=/d /requirements/upper-constraints.txt \
     && {{ macros.install_pip(horizon_pip_packages | customizable("pip_packages")) }} \
     && mkdir -p /etc/openstack-dashboard \
     && cp -r /horizon/openstack_dashboard/conf/* /etc/openstack-dashboard/ \

docker/ironic/ironic-api/extend_start.sh

@@ -25,6 +25,7 @@ if [[ "$(whoami)" == 'root' ]]; then
     if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
         # Loading Apache2 ENV variables
         . /etc/apache2/envvars
+        install -d /var/run/apache2/
         rm -rf /var/run/apache2/*
     else
         rm -rf /var/run/httpd/* /run/httpd/* /tmp/httpd*

docker/ironic/ironic-conductor/Dockerfile.j2

@@ -13,6 +13,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'e2fsprogs',
             'gdisk',
             'ipmitool',
+            'openssh-clients',
             'openstack-ironic-conductor',
             'openstack-ironic-staging-drivers',
             'parted',
@@ -67,6 +68,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'gdisk',
             'ipmitool',
             'ironic-conductor',
+            'openssh-client',
             'parted',
             'psmisc',
             'python3-ironic-inspector-client',
@@ -88,6 +90,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'fuse',
             'gdisk',
             'ipmitool',
+            'openssh-clients',
             'parted',
             'psmisc',
             'qemu-img-ev',
@@ -97,10 +100,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         ] %}
         {% if distro_python_version.startswith('3') %}
         {% set ironic_conductor_packages = ironic_conductor_packages + [
+            'python3-systemd',
             'systemd-udev',
         ] %}
         {% else %}
         {% set ironic_conductor_packages = ironic_conductor_packages + [
+            'systemd-python',
             'systemd',
         ] %}
         {% endif %}
@@ -111,8 +116,10 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'e2fsprogs',
             'gdisk',
             'ipmitool',
+            'openssh-client',
             'parted',
             'psmisc',
+            'python-systemd',
             'qemu-utils',
             'shellinabox',
             'udev',

docker/ironic/ironic-pxe/extend_start.sh

@@ -45,6 +45,7 @@ fi
 if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
     # Loading Apache2 ENV variables
     . /etc/apache2/envvars
+    install -d /var/run/apache2/
     rm -rf /var/run/apache2/*
 else
     rm -rf /var/run/httpd/* /run/httpd/* /tmp/httpd*

docker/keystone/keystone-fernet/fetch_fernet_tokens.py

@@ -43,12 +43,12 @@ def has_file(filename_path):
 
 
 def num_tokens():
-    _, _, files = os.walk(TOKEN_PATH).next()
+    _, _, files = next(os.walk(TOKEN_PATH))
     return len(files)
 
 
 def tokens_populated(expected):
-    return num_tokens() == int(expected)
+    return num_tokens() >= int(expected)
 
 
 def token_stale(seconds, filename='0'):
@@ -71,7 +71,7 @@ def main():
                         help='Filename of token to check',
                         default='0')
     parser.add_argument('-n', '--number',
-                        help='Number of tokens that should exist',
+                        help='Minimum number of tokens that should exist',
                         required=True)
     args = parser.parse_args()
 

docker/keystone/keystone/extend_start.sh

@@ -5,6 +5,7 @@
 if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
     # Loading Apache2 ENV variables
     . /etc/apache2/envvars
+    install -d /var/run/apache2/
     rm -rf /var/run/apache2/*
 else
     rm -rf /var/run/httpd/* /run/httpd/* /tmp/httpd*

docker/keystone/keystone/keystone_bootstrap.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -o pipefail
+
 # NOTE(SamYaple): Kolla needs to wraps `keystone-manage bootstrap` to ensure
 # any change is reported correctly for idempotency. This script will exit with
 # valid json that can be parsed with information about if the task has failed
@@ -29,7 +31,11 @@ function exit_json {
 }
 
 changed="false"
-keystone_bootstrap=$(keystone-manage bootstrap --bootstrap-username "${USERNAME}" --bootstrap-password "${PASSWORD}" --bootstrap-project-name "${PROJECT}" --bootstrap-role-name "${ROLE}" --bootstrap-admin-url "${ADMIN_URL}" --bootstrap-internal-url "${INTERNAL_URL}" --bootstrap-public-url "${PUBLIC_URL}" --bootstrap-service-name "keystone" --bootstrap-region-id "${REGION}" 2>&1)
+# NOTE(mgoddard): pipe through cat -v to remove unprintable control characters
+# which prevent JSON decoding.
+# NOTE(yoctozepto): also apply sed to escape double quotation marks
+# and backslashes
+keystone_bootstrap=$(keystone-manage bootstrap --bootstrap-username "${USERNAME}" --bootstrap-password "${PASSWORD}" --bootstrap-project-name "${PROJECT}" --bootstrap-role-name "${ROLE}" --bootstrap-admin-url "${ADMIN_URL}" --bootstrap-internal-url "${INTERNAL_URL}" --bootstrap-public-url "${PUBLIC_URL}" --bootstrap-service-name "keystone" --bootstrap-region-id "${REGION}" 2>&1 | cat -v | sed 's/\\/\\\\/g' | sed 's/"/\\"/g')
 if [[ $? != 0 ]]; then
     fail_json "${keystone_bootstrap}"
 fi

docker/kolla-toolbox/Dockerfile.j2

@@ -54,7 +54,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         'virtualenv'
     ] %}
 
-RUN {{ macros.install_pip(kolla_toolbox_pip_virtualenv_packages|customizable("pip_packages"), constraints=false) }} \
+RUN {{ macros.install_pip(kolla_toolbox_pip_virtualenv_packages | customizable("pip_virtualenv_packages"), constraints=false) }} \
     && virtualenv --system-site-packages {{ virtualenv_path }}
 
 ENV PATH {{ virtualenv_path }}/bin:$PATH
@@ -64,9 +64,14 @@ ENV PATH {{ virtualenv_path }}/bin:$PATH
         '"cmd2<0.9.0"',
         'influxdb',
         'mysqlclient',
+        '"openstacksdk<0.28.0"',
         'os-client-config==1.29.0',
         'pbr==4.0.0',
         'pymongo',
+        '"python-cinderclient<5.0.0"',
+        '"python-glanceclient<2.17.0"',
+        '"python-keystoneclient<3.20.0"',
+        '"python-novaclient<14.0.0"',
         'python-openstackclient==3.14.0',
         'pytz',
         'pyudev',

docker/manila/manila-api/extend_start.sh

@@ -15,6 +15,7 @@ if [[ "$(whoami)" == 'root' ]]; then
     if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
         # Loading Apache2 ENV variables
         . /etc/apache2/envvars
+        install -d /var/run/apache2/
         rm -rf /var/run/apache2/*
     else
         rm -rf /var/run/httpd/* /run/httpd/* /tmp/httpd*

docker/manila/manila-share/Dockerfile.j2

@@ -17,6 +17,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         {% set manila_share_packages = [
             'manila-share',
             'ceph-common',
+            'python3-cephfs',
             'python3-rados',
             'python3-rbd',
             'sqlite3'

docker/mariadb/Dockerfile.j2

@@ -27,6 +27,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     ] %}
     {% if base_distro == 'debian' %}
         {% set mariadb_packages = mariadb_packages + [
+            'mariadb-backup',
             'mariadb-server'
         ] %}
     {% else %}
@@ -52,6 +53,9 @@ RUN chmod 755 /usr/local/bin/kolla_extend_start \
 RUN mkdir -p /var/run/mysqld && chown mysql /var/run/mysqld && chmod 755 /var/run/mysqld
 {% endif %}
 
+COPY backup.sh /usr/local/bin/kolla_mariadb_backup.sh
+RUN chmod 755 /usr/local/bin/kolla_mariadb_backup.sh
+
 {% if use_dumb_init %}
 {% block mariadb_entrypoint %}
 # NOTE(mgoddard): Override the dumb-init arguments to avoid passing

docker/mariadb/backup.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+# Execute a full backup
+backup_full() {
+    echo "Taking a full backup"
+    mariabackup \
+        --defaults-file=/etc/mysql/my.cnf \
+        --backup \
+        --stream=xbstream \
+        --history=$(date +%d-%m-%Y) | gzip > \
+        $BACKUP_DIR/mysqlbackup-$(date +%d-%m-%Y-%s).qp.xbc.xbs.gz
+}
+
+# Execute an incremental backup
+backup_incremental() {
+    echo "Taking an incremental backup"
+    mariabackup \
+        --defaults-file=/etc/mysql/my.cnf \
+        --backup \
+        --stream=xbstream \
+        --incremental-history-name=$(date +%d-%m-%Y) \
+        --history=$(date +%d-%m-%Y) | gzip > \
+        $BACKUP_DIR/incremental-$(date +%H)-mysqlbackup-$(date +%d-%m-%Y-%s).qp.xbc.xbs.gz
+}
+
+BACKUP_DIR=/backup/
+cd $BACKUP_DIR
+
+if [ -n $BACKUP_TYPE ]; then
+    case $BACKUP_TYPE in
+        "full")
+        backup_full
+        ;;
+        "incremental")
+        backup_incremental
+        ;;
+        *)
+        echo "Only full or incremental options are supported."
+        exit 1
+        ;;
+    esac
+else
+    echo "You need to specify either full or incremental backup options."
+    exit 1
+fi

docker/mistral/mistral-api/extend_start.sh

@@ -15,6 +15,7 @@ if [[ "$(whoami)" == 'root' ]]; then
     if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
         # Loading Apache2 ENV variables
         . /etc/apache2/envvars
+        install -d /var/run/apache2/
         rm -rf /var/run/apache2/*
     else
         rm -rf /var/run/httpd/* /run/httpd/* /tmp/httpd*