stackhpc · jovial · Oct 22, 2019 · Jun 19, 2019 · May 6, 2019 · Jul 19, 2019
diff --git a/.yamllint b/.yamllint
@@ -1,3 +1,4 @@
+---
 extends: default
 ignore: |
   .tox/

diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml
@@ -40,8 +40,8 @@
 - job:
     name: kolla-base
     parent: base
-    timeout: 10200
-    post-timeout: 7200
+    timeout: 10800
+    post-timeout: 10800
     pre-run: tests/playbooks/pre.yml
     run: tests/playbooks/run.yml
     post-run: tests/playbooks/post.yml

diff --git a/.zuul.d/centos.yaml b/.zuul.d/centos.yaml
@@ -8,7 +8,8 @@
         - kolla-ansible-centos-binary
         - kolla-ansible-centos-source-upgrade
         - tripleo-build-containers-centos-7:
-            voting: true
+            # FIXME(yoctozepto): set to voting when TripleO CI is fixed
+            voting: false
             files:
               - ^.zuul.d/centos.yaml$
               - ^docker/.*$
@@ -24,15 +25,6 @@
       queue: kolla
       jobs:
         - kolla-build-centos-source
-        - tripleo-build-containers-centos-7:
-            voting: true
-            files:
-              - ^.zuul.d/centos.yaml$
-              - ^docker/.*$
-              - ^kolla/.*$
-              - ^requirements.txt$
-              - ^setup.cfg$
-              - ^setup.py$
     periodic:
       jobs:
         - kolla-publish-centos-source

diff --git a/docker/aodh/aodh-api/Dockerfile.j2 b/docker/aodh/aodh-api/Dockerfile.j2
@@ -12,6 +12,10 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         {% set aodh_api_packages = ['aodh-api'] %}
     {% endif %}
 {{ macros.install_packages(aodh_api_packages | customizable("packages")) }}
+
+    {% if base_package_type == 'deb' %}
+RUN rm -rf /etc/apache2/sites-enabled/aodh-api.conf
+    {% endif %}
 {% endif %}
 
 COPY extend_start.sh /usr/local/bin/kolla_aodh_extend_start

diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2
@@ -185,7 +185,7 @@ RUN {{ macros.install_packages( ['tar', 'yum-utils', 'https://dl.fedoraproject.o
         http://mirror.centos.org/centos-7/7/extras/x86_64/Packages/centos-release-nfs-ganesha28-1.0-2.el7.centos.noarch.rpm \
         http://mirror.centos.org/centos-7/7/extras/x86_64/Packages/centos-release-ceph-nautilus-1.2-2.el7.centos.noarch.rpm \
         http://mirror.centos.org/centos-7/7/extras/x86_64/Packages/centos-release-opstools-1-8.el7.noarch.rpm \
-        http://mirror.centos.org/centos-7/7/extras/x86_64/Packages/centos-release-qemu-ev-1.0-3.el7.centos.noarch.rpm \
+        http://mirror.centos.org/centos-7/7/extras/x86_64/Packages/centos-release-qemu-ev-1.0-4.el7.centos.noarch.rpm \
         http://mirror.centos.org/centos-7/7/extras/x86_64/Packages/centos-release-virt-common-1-1.el7.centos.noarch.rpm \
         http://mirror.centos.org/centos-7/7/extras/x86_64/Packages/centos-release-storage-common-2-2.el7.centos.noarch.rpm \
     && sed -i 's/\$releasever/7/g' /etc/yum.repos.d/CentOS-*.repo \
@@ -353,7 +353,9 @@ COPY apt_preferences.{{ base_distro }} /etc/apt/preferences
     {% set remote_apt_keys = [
       'http://obs.linaro.org/ERP:/18.06/Debian_9/Release.key',
       'https://download.docker.com/linux/debian/gpg',
-      'https://packages.grafana.com/gpg.key'
+      'https://packages.grafana.com/gpg.key',
+      'http://buster-stein.debian.net/debian/dists/pubkey.gpg',
+      'https://packages.treasuredata.com/GPG-KEY-td-agent',
     ] %}
     {% set base_apt_packages = base_apt_packages +
       ['sudo',]

diff --git a/docker/base/nfs_ganesha.repo b/docker/base/nfs_ganesha.repo
diff --git a/docker/base/sources.list.debian b/docker/base/sources.list.debian
@@ -11,3 +11,10 @@ deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main
 
 # main docker repo
 deb https://download.docker.com/linux/debian buster stable
+
+# Buster - Stein repos
+deb http://buster-stein.debian.net/debian buster-stein-backports main
+deb http://buster-stein.debian.net/debian buster-stein-backports-nochange main
+
+# td-agent for fluentd
+deb http://packages.treasuredata.com/3/debian/stretch stretch contrib
diff --git a/docker/ceph/ceph-mon/extend_start.sh b/docker/ceph/ceph-mon/extend_start.sh
@@ -22,7 +22,7 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
 
     # Generating initial keyrings and monmap
     ceph-authtool --create-keyring "${KEYRING_MON}" --gen-key -n mon. --cap mon 'allow *'
-    ceph-authtool --create-keyring "${KEYRING_ADMIN}" --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow' --cap mgr 'allow *'
+    ceph-authtool --create-keyring "${KEYRING_ADMIN}" --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
     ceph-authtool --create-keyring "${KEYRING_RGW}" --gen-key -n client.radosgw.gateway --cap osd 'allow rwx' --cap mon 'allow rwx'
     ceph-authtool "${KEYRING_MON}" --import-keyring "${KEYRING_ADMIN}"
     ceph-authtool "${KEYRING_MON}" --import-keyring "${KEYRING_RGW}"

diff --git a/docker/ceph/ceph-osd/extend_start.sh b/docker/ceph/ceph-osd/extend_start.sh
@@ -7,6 +7,42 @@ if [[ $(stat -c %a /var/log/kolla/ceph) != "755" ]]; then
     chmod 755 /var/log/kolla/ceph
 fi
 
+# Inform the os about partition table changes
+function partprobe_device {
+    local device=$1
+    udevadm settle --timeout=600
+    flock -s ${device} partprobe ${device}
+    udevadm settle --timeout=600
+}
+
+# In some cases, the disk partition will not appear immediately, so check every
+# 1s, try up to 10 times. In general, this interval is enough.
+function wait_partition_appear {
+    local dev_part=$1
+    local part_name=$(echo ${dev_part} | awk -F '/' '{print $NF}')
+    for(( i=1; i<11; i++ )); do
+        flag=$(ls /dev | awk '/'"${part_name}"'/{print $0}' | wc -l)
+        if [[ "${flag}" -eq 0 ]]; then
+            echo "sleep 1 waits for the partition ${dev_part} to appear: ${i}"
+            sleep 1
+        else
+            return 0
+        fi
+    done
+    echo "The device /dev/${dev_part} does not appear within the limited time 10s."
+    exit 1
+}
+
+# Few storage device like loop or NVMe, wiil add "p" between disk & partition
+# name if disk layout is end with number. This function will fix to correct format.
+function part_name_checker {
+    if [[ $1 =~ .*[0-9] ]]; then
+        echo ${1}p${2}
+    else
+        echo ${1}${2}
+    fi
+}
+
 # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
@@ -22,45 +58,26 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
     if [[ "${USE_EXTERNAL_JOURNAL}" == "False" ]]; then
         # Formatting disk for ceph
         if [[ "${OSD_STORETYPE}" == "bluestore" ]]; then
-            if [[ "${OSD_BS_DEV}" =~ "/dev/loop" ]]; then
-                sgdisk --zap-all -- "${OSD_BS_DEV}""p${OSD_BS_PARTNUM}"
-            else
-                sgdisk --zap-all -- "${OSD_BS_DEV}""${OSD_BS_PARTNUM}"
-            fi
+            sgdisk --zap-all -- "$(part_name_checker $OSD_BS_DEV $OSD_BS_PARTNUM)"
 
             if [ -n "${OSD_BS_BLK_DEV}" ] && [ "${OSD_BS_DEV}" != "${OSD_BS_BLK_DEV}" ] && [ -n "${OSD_BS_BLK_PARTNUM}" ]; then
-                if [[ "${OSD_BS_BLK_DEV}" =~ "/dev/loop" ]]; then
-                    sgdisk --zap-all -- "${OSD_BS_BLK_DEV}""p${OSD_BS_BLK_PARTNUM}"
-                else
-                    sgdisk --zap-all -- "${OSD_BS_BLK_DEV}""${OSD_BS_BLK_PARTNUM}"
-                fi
+                sgdisk --zap-all -- "$(part_name_checker ${OSD_BS_BLK_DEV} ${OSD_BS_BLK_PARTNUM})"
             else
                 sgdisk --zap-all -- "${OSD_BS_DEV}"
                 sgdisk --new=1:0:+100M --mbrtogpt -- "${OSD_BS_DEV}"
                 sgdisk --largest-new=2 --mbrtogpt -- "${OSD_BS_DEV}"
-                partprobe || true
+                partprobe_device "${OSD_BS_DEV}"
 
-                if [[ "${OSD_BS_DEV}" =~ "/dev/loop" ]]; then
-                    sgdisk --zap-all -- "${OSD_BS_DEV}"p2
-                else
-                    sgdisk --zap-all -- "${OSD_BS_DEV}"2
-                fi
+                wait_partition_appear "$(part_name_checker $OSD_BS_DEV 2)"
+                sgdisk --zap-all -- "$(part_name_checker $OSD_BS_DEV 2)"
             fi
 
             if [ -n "${OSD_BS_WAL_DEV}" ] && [ "${OSD_BS_BLK_DEV}" != "${OSD_BS_WAL_DEV}" ] && [ -n "${OSD_BS_WAL_PARTNUM}" ]; then
-                if [[ "${OSD_BS_WAL_DEV}" =~ "/dev/loop" ]]; then
-                    sgdisk --zap-all -- "${OSD_BS_WAL_DEV}""p${OSD_BS_WAL_PARTNUM}"
-                else
-                    sgdisk --zap-all -- "${OSD_BS_WAL_DEV}""${OSD_BS_WAL_PARTNUM}"
-                fi
+                sgdisk --zap-all -- "$(part_name_checker $OSD_BS_WAL_DEV $OSD_BS_WAL_PARTNUM)"
             fi
 
             if [ -n "${OSD_BS_DB_DEV}" ] && [ "${OSD_BS_BLK_DEV}" != "${OSD_BS_DB_DEV}" ] && [ -n "${OSD_BS_DB_PARTNUM}" ]; then
-                if [[ "${OSD_BS_DB_DEV}" =~ "/dev/loop" ]]; then
-                    sgdisk --zap-all -- "${OSD_BS_DB_DEV}""p${OSD_BS_DB_PARTNUM}"
-                else
-                    sgdisk --zap-all -- "${OSD_BS_DB_DEV}""${OSD_BS_DB_PARTNUM}"
-                fi
+                sgdisk --zap-all -- "$(part_name_checker $OSD_BS_DB_DEV $OSD_BS_DB_PARTNUM)"
             fi
         else
             sgdisk --zap-all -- "${OSD_DEV}"
@@ -78,13 +95,8 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
         OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
         mkdir -p "${OSD_DIR}"
 
-        if [[ "${OSD_BS_DEV}" =~ "/dev/loop" ]]; then
-            mkfs.xfs -f "${OSD_BS_DEV}""p${OSD_BS_PARTNUM}"
-            mount "${OSD_BS_DEV}""p${OSD_BS_PARTNUM}" "${OSD_DIR}"
-        else
-            mkfs.xfs -f "${OSD_BS_DEV}""${OSD_BS_PARTNUM}"
-            mount "${OSD_BS_DEV}""${OSD_BS_PARTNUM}" "${OSD_DIR}"
-        fi
+        mkfs.xfs -f "$(part_name_checker $OSD_BS_DEV $OSD_BS_PARTNUM)"
+        mount "$(part_name_checker $OSD_BS_DEV $OSD_BS_PARTNUM)" "${OSD_DIR}"
 
         # This will through an error about no key existing. That is normal. It then
         # creates the key in the next step.
@@ -130,11 +142,8 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
 
         ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow profile osd' -i "${OSD_DIR}/keyring"
 
-        if [[ "${OSD_BS_DEV}" =~ "/dev/loop" ]]; then
-            umount "${OSD_BS_DEV}""p${OSD_BS_PARTNUM}"
-        else
-            umount "${OSD_BS_DEV}""${OSD_BS_PARTNUM}"
-        fi
+        umount "$(part_name_checker $OSD_BS_DEV $OSD_BS_PARTNUM)"
+
     else
         OSD_ID=$(ceph osd create)
         OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"

diff --git a/docker/cyborg/cyborg-agent/Dockerfile.j2 b/docker/cyborg/cyborg-agent/Dockerfile.j2
@@ -4,6 +4,10 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 {% block cyborg_agent_header %}{% endblock %}
 {% import "macros.j2" as macros with context %}
 
+{% set cyborg_agent_packages = [
+    'pciutils',
+] %}
+
 {% if install_type == 'binary' %}
 
 RUN echo '{{ install_type }} not yet available for {{ base_distro }}'  \
@@ -17,6 +21,8 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}'  \
 
 {% endif %}
 
+{{ macros.install_packages(cyborg_agent_packages | customizable("packages")) }}
+
 RUN {{ macros.install_pip(cyborg_agent_pip_packages | customizable("pip_packages")) }} 
 
 {% if base_package_type == 'rpm' %}

diff --git a/docker/elasticsearch/Dockerfile.j2 b/docker/elasticsearch/Dockerfile.j2
@@ -21,7 +21,7 @@ ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk/
         'elasticsearch',
     ] %}
 
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-{{debian_arch}}/
+ENV JAVA_HOME /usr/lib/jvm/java-{{java_version}}-openjdk-{{debian_arch}}/
 
 {% endif %}
 

diff --git a/docker/fluentd/Dockerfile.j2 b/docker/fluentd/Dockerfile.j2
@@ -21,7 +21,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         'ruby-devel',
         'rubygem-fluent-plugin-elasticsearch',
         'rubygem-fluent-plugin-grok-parser',
-        'rubygem-fluent-plugin-kubernetes_metadata_filter',
         'rubygem-fluent-plugin-rewrite-tag-filter',
         'rubygem-fluent-plugin-secure-forward'
     ] %}
@@ -65,7 +64,6 @@ RUN chmod 755 /usr/local/bin/kolla_extend_start
         'fluent-plugin-elasticsearch',
         'fluent-plugin-grep',
         'fluent-plugin-grok-parser:2.1.4',
-        'fluent-plugin-kubernetes_metadata_filter',
         'fluent-plugin-parser',
         'fluent-plugin-rewrite-tag-filter:2.0.0',
         'fluent-plugin-secure-forward',
@@ -75,7 +73,7 @@ RUN chmod 755 /usr/local/bin/kolla_extend_start
 {{ macros.install_fluent_plugins(fluentd_plugins | customizable("plugins")) }}
 
 # Build and install Fluentd output plugin for Monasca Log API
-ARG monasca_output_plugin_tag=0.1.0
+ARG monasca_output_plugin_tag=0.1.1
 ARG monasca_output_plugin_url=https://github.com/monasca/fluentd-monasca/archive/$monasca_output_plugin_tag.tar.gz
 ADD $monasca_output_plugin_url /tmp/fluentd-monasca.tar.gz
 RUN tar -xvf /tmp/fluentd-monasca.tar.gz -C /tmp \

diff --git a/docker/gnocchi/gnocchi-api/Dockerfile.j2 b/docker/gnocchi/gnocchi-api/Dockerfile.j2
@@ -15,6 +15,10 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         {% set gnocchi_api_packages = ['gnocchi-api'] %}
     {% endif %}
 {{ macros.install_packages(gnocchi_api_packages | customizable("packages")) }}
+
+    {% if base_package_type == 'deb' %}
+RUN rm -rf /etc/apache2/sites-enabled/gnocchi-api.conf
+    {% endif %}
 {% endif %}
 
 COPY extend_start.sh /usr/local/bin/kolla_gnocchi_extend_start

diff --git a/docker/horizon/Dockerfile.j2 b/docker/horizon/Dockerfile.j2
@@ -67,6 +67,7 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
                 'python3-designate-dashboard',
                 'python3-heat-dashboard',
                 'python3-manila-ui',
+                'python3-octavia-dashboard',
                 'python3-sahara-dashboard',
                 'python3-trove-dashboard',
                 'tzdata'
@@ -83,6 +84,8 @@ RUN echo > /etc/apache2/ports.conf \
     && cp /usr/share/openstack-dashboard/openstack_dashboard/conf/*.json /etc/openstack-dashboard \
     && cp /usr/share/openstack-dashboard/manage.py /usr/bin/manage.py \
     && rm /etc/apache2/conf-enabled/openstack-dashboard.conf \
+    && rm /etc/openstack-dashboard/local_settings.py \
+    && ln -s /etc/openstack-dashboard/local_settings /etc/openstack-dashboard/local_settings.py \
     && for locale in /usr/lib/python{{distro_python_version}}/site-packages/*/locale; do \
            (cd ${locale%/*} && /usr/bin/python3 /usr/bin/manage.py compilemessages) \
        done

diff --git a/docker/kafka/Dockerfile.j2 b/docker/kafka/Dockerfile.j2
@@ -12,8 +12,15 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         'java-1.8.0-openjdk-headless',
     ] %}
 {% elif base_package_type == 'deb' %}
+
+    {% if base_distro == 'debian' %}
+        {% set java_version = '11' %}
+    {% elif base_distro == 'ubuntu' %}
+        {% set java_version = '8' %}
+    {% endif %}
+
     {% set kafka_packages = [
-        'openjdk-8-jre-headless',
+        'openjdk-' + java_version + '-jre-headless',
     ] %}
 {% endif %}
 

diff --git a/docker/logstash/Dockerfile.j2 b/docker/logstash/Dockerfile.j2
@@ -15,12 +15,19 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk/
 
 {% elif base_package_type == 'deb' %}
+
+    {% if base_distro == 'debian' %}
+        {% set java_version = '11' %}
+    {% elif base_distro == 'ubuntu' %}
+        {% set java_version = '8' %}
+    {% endif %}
+
     {% set logstash_packages = [
-        'openjdk-8-jre-headless',
+        'openjdk-' + java_version + '-jre-headless',
         'logrotate'
     ] %}
 
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-{{debian_arch}}/
+ENV JAVA_HOME /usr/lib/jvm/java-{{java_version}}-openjdk-{{debian_arch}}/
 
 {% endif %}
 

diff --git a/docker/monasca/monasca-agent/Dockerfile.j2 b/docker/monasca/monasca-agent/Dockerfile.j2
@@ -15,7 +15,8 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \
 ADD monasca-agent-archive /monasca-agent-source
 
 {% set monasca_agent_pip_packages = [
-    '/monasca-agent'
+    '/monasca-agent',
+    'prometheus_client'
 ] %}
 
 RUN ln -s monasca-agent-source/* monasca-agent \

diff --git a/docker/monasca/monasca-grafana/Dockerfile.j2 b/docker/monasca/monasca-grafana/Dockerfile.j2
@@ -48,7 +48,9 @@ ARG monasca_grafana_version=grafana4
 ARG monasca_grafana_url=https://github.com/monasca/grafana/archive/$monasca_grafana_version.tar.gz
 
 {% block monasca_grafana_install %}
-RUN gem install rake fpm \
+# NOTE(mgoddard): Pinning rake because rake 13.0.0 depends on Ruby 2.2, and CentOS 7 only provides
+# Ruby 2.0.
+RUN gem install rake:"~>12" fpm \
     && curl -sSL -o /tmp/monasca-grafana.tgz ${monasca_grafana_url} \
     && mkdir -p ${monasca_grafana_build_path} \
     && tar --strip 1 -xvf /tmp/monasca-grafana.tgz -C ${monasca_grafana_build_path} \

diff --git a/docker/monasca/monasca-thresh/Dockerfile.j2 b/docker/monasca/monasca-thresh/Dockerfile.j2
@@ -24,12 +24,19 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \
 ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk/
 
 {% elif base_package_type == 'deb' %}
+
+    {% if base_distro == 'debian' %}
+        {% set java_version = '11' %}
+    {% elif base_distro == 'ubuntu' %}
+        {% set java_version = '8' %}
+    {% endif %}
+
     {% set monasca_thresh_packages = [
-        'openjdk-8-jdk-headless',
+        'openjdk-' + java_version + '-jdk-headless',
         'maven',
     ] %}
 
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-{{debian_arch}}/
+ENV JAVA_HOME /usr/lib/jvm/java-{{java_version}}-openjdk-{{debian_arch}}/
 
 {% endif %}