From 627beae3b8ddd1de3ddc34637f3e29efcdd53681 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Tue, 28 Apr 2020 14:59:45 +0200 Subject: [PATCH 01/10] drop tripleo-common-tempest-plugin This is not needed plugin without any branches or releases. And wants Python 3 while we have Py2 for CentOS 7. Change-Id: I9cab91b82941ea125b16d19e68722ce41a1f11f2 14:41 < hrw> is tripleo-common-tempest-plugin your stuff? 14:43 <@EmilienM> yes and it does nothing. I thought we removed that? 14:43 <@EmilienM> apparently it's still here https://github.com/openstack/tripleo-common-tempest-plugin 14:43 <@EmilienM> we should burn it, weshay|ruck ^ 14:44 < hrw> EmilienM: also fine to drop from train? 14:44 <@EmilienM> burn it everywhere you can 14:44 < hrw> THANKS 14:44 <@EmilienM> it has never been useful 14:44 <@EmilienM> and people who started it left 14:48 <@weshay|ruck> EmilienM, yup --- kolla/common/config.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/kolla/common/config.py b/kolla/common/config.py index acdc790931..7da753bbda 100755 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -749,10 +749,6 @@ 'type': 'url', 'location': ('$tarballs_base/telemetry-tempest-plugin/' 'telemetry_tempest_plugin-0.3.0.tar.gz')}, - 'tempest-plugin-tripleo-common': { - 'type': 'url', - 'location': ('$tarballs_base/tripleo-common-tempest-plugin/' - 'tripleo-common-tempest-plugin-master.tar.gz')}, 'tempest-plugin-trove': { 'type': 'url', 'location': ('$tarballs_base/trove-tempest-plugin/' From 0cf37e0091bf51f169436cfb1c4290de892042aa Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 1 May 2020 19:10:28 +0100 Subject: [PATCH 02/10] Bump versions for Stein Change-Id: Ic06bdcf30b7df7447b1bcbf2f688a9d148a59c67 --- kolla/common/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kolla/common/config.py b/kolla/common/config.py index 7da753bbda..365e469ebd 100755 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -383,7 +383,7 @@ 'horizon': { 'type': 'url', 'location': ('$tarballs_base/horizon/' - 'horizon-15.2.0.tar.gz')}, + 'horizon-15.3.0.tar.gz')}, 'horizon-plugin-blazar-dashboard': { 'type': 'url', 'location': ('$tarballs_base/blazar-dashboard/' @@ -523,7 +523,7 @@ 'manila-base': { 'type': 'url', 'location': ('$tarballs_base/manila/' - 'manila-8.1.0.tar.gz')}, + 'manila-8.1.2.tar.gz')}, 'mistral-base': { 'type': 'url', 'location': ('$tarballs_base/mistral/' @@ -656,7 +656,7 @@ 'nova-base': { 'type': 'url', 'location': ('$tarballs_base/nova/' - 'nova-19.1.0.tar.gz')}, + 'nova-19.2.0.tar.gz')}, 'nova-base-plugin-blazar': { 'type': 'url', 'location': ('$tarballs_base/blazar-nova/' From 0f7cc7c95d107f6eb5af9d68b4f01464414aefe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Thu, 12 Mar 2020 17:24:10 +0100 Subject: [PATCH 03/10] CI: Ignore .zuul.d Zuul handles jobs updates just fine, no need to run kolla build jobs when touching .zuul.d for different reasons. Change-Id: I201a194fa473a63b880cefc00febb543b40bcec8 (cherry picked from commit a0087490fffa11e72eaafc490ae67954f0966831) --- .zuul.d/base.yaml | 1 + .zuul.d/centos.yaml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml index 8847eb55c9..9a13391ce2 100644 --- a/.zuul.d/base.yaml +++ b/.zuul.d/base.yaml @@ -54,6 +54,7 @@ - ^releasenotes/.*$ - ^specs/.*$ - ^test-requirements.txt$ + - ^\.zuul\.d/ vars: publisher: false extra-vars: diff --git a/.zuul.d/centos.yaml b/.zuul.d/centos.yaml index 98a6d8ab50..fcbd6ea9ae 100644 --- a/.zuul.d/centos.yaml +++ b/.zuul.d/centos.yaml @@ -11,7 +11,6 @@ # FIXME(yoctozepto): set to voting when TripleO CI is fixed voting: false files: - - ^.zuul.d/centos.yaml$ - ^docker/.*$ - ^kolla/.*$ - ^requirements.txt$ From 4b77682911616a6cfac96df3695f0d7896f89b50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Sat, 23 May 2020 11:19:12 +0200 Subject: [PATCH 04/10] CI: Ignore more This aligns Kolla with Kolla Ansible. Change-Id: I02ee643d0f46e6ff1ed376362b4ee35ddadf7960 (cherry picked from commit 8500eaee2a030c2720a008f8a9a073ec20fad70e) (cherry picked from commit d3587d44d7c631fcba63df9dc199280988ae6551) (cherry picked from commit c88e60d6805e0d32ac404bff863f6b541ffa7b64) --- .gitignore | 3 +++ .zuul.d/base.yaml | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 4f71aa718f..2fe85e16cb 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,9 @@ cover/ .project .pydevproject +# Files generated by Visual Studio Code +.vscode/ + # Files created by reno build releasenotes/build diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml index 9a13391ce2..21c1ee8f50 100644 --- a/.zuul.d/base.yaml +++ b/.zuul.d/base.yaml @@ -55,6 +55,10 @@ - ^specs/.*$ - ^test-requirements.txt$ - ^\.zuul\.d/ + - ^\..+ + - ^contrib/ + - ^LICENSE$ + - ^tox\.ini$ vars: publisher: false extra-vars: From fca4a088a49bbd8ee7ed6ce8b0b128df7b62399f Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 4 Jun 2020 12:05:12 +0100 Subject: [PATCH 05/10] Fix multiple issues Both issues affect Train and earlier releases. 1. Fix monasca-grafana by pinning more Ruby gems childprocess 3.0.0 and ffi 1.13.0 depend on Ruby 2.3. 2. Bump bifrost to 6.0.4 Bifrost had a fix to stop using the master branch of DIB which dropped support for Python 2. 3. Drop docker-client from OracleLinux sensu-client image The package fails to install due to a missing dependency on subscription-manager. Change-Id: Ida7e20833360bbca69c6aafc4f5d7cb375bb106f Closes-Bug: #1882070 (cherry picked from commit 3784d32d9ceacf0dad8057092752f1ae7be37ed3) --- docker/monasca/monasca-grafana/Dockerfile.j2 | 3 ++- docker/sensu/sensu-client/Dockerfile.j2 | 7 ++++++- kolla/common/config.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docker/monasca/monasca-grafana/Dockerfile.j2 b/docker/monasca/monasca-grafana/Dockerfile.j2 index 295f37ae2f..2309aff2a8 100644 --- a/docker/monasca/monasca-grafana/Dockerfile.j2 +++ b/docker/monasca/monasca-grafana/Dockerfile.j2 @@ -53,7 +53,8 @@ ARG monasca_grafana_url=https://github.com/monasca/grafana/archive/$monasca_graf # NOTE(yoctozepto): Update npm to 6.x version to avoid issues with metadata parsing. # NPM installs itself in /usr/local/bin (the default in Ubuntu) which is not in the PATH. # This is forced for all distros to avoid conflicts with native packages. -RUN gem install rake:"~>12" fpm \ +# NOTE(mgoddard): childprocess 3.0.0 and ffi 1.13.0 depend on Ruby 2.3. +RUN gem install rake:"~>12" ffi:"<1.13.0" childprocess:"<2.0.0" fpm \ && curl -sSL -o /tmp/monasca-grafana.tgz ${monasca_grafana_url} \ && mkdir -p ${monasca_grafana_build_path} \ && tar --strip 1 -xvf /tmp/monasca-grafana.tgz -C ${monasca_grafana_build_path} \ diff --git a/docker/sensu/sensu-client/Dockerfile.j2 b/docker/sensu/sensu-client/Dockerfile.j2 index 9ccd4f3ae2..90f6bff452 100644 --- a/docker/sensu/sensu-client/Dockerfile.j2 +++ b/docker/sensu/sensu-client/Dockerfile.j2 @@ -9,13 +9,18 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% set sensu_client_packages = [ 'ceph-common', 'cyrus-sasl-devel', - 'docker-client', 'gcc-c++', 'make', 'mariadb', 'ntp', 'ruby-devel' ] %} + {% if base_distro != 'oraclelinux' %} + # NOTE(mgoddard): Installing docker-client fails on OracleLinux. + {% set sensu_client_packages = sensu_client_packages + [ + 'docker-client', + ] %} + {% endif %} {% if distro_python_version.startswith('3') %} {% set sensu_client_packages = sensu_client_packages + [ diff --git a/kolla/common/config.py b/kolla/common/config.py index 365e469ebd..6d4ddab379 100755 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -315,7 +315,7 @@ 'bifrost-base': { 'type': 'url', 'location': ('$tarballs_base/bifrost/' - 'bifrost-6.0.3.tar.gz')}, + 'bifrost-6.0.4.tar.gz')}, 'blazar-base': { 'type': 'url', 'location': ('$tarballs_base/blazar/' From 8cce9293a42afca7a99f32090d4f55b1cf7310a7 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 29 May 2020 17:23:29 +0100 Subject: [PATCH 06/10] Bump versions for Stein Change-Id: Iaa16955b9b4b078901ce17934721b7780cc43b37 --- kolla/common/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kolla/common/config.py b/kolla/common/config.py index 6d4ddab379..bcb5af4138 100755 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -379,7 +379,7 @@ 'heat-base': { 'type': 'url', 'location': ('$tarballs_base/heat/' - 'openstack-heat-12.0.0.tar.gz')}, + 'openstack-heat-12.1.0.tar.gz')}, 'horizon': { 'type': 'url', 'location': ('$tarballs_base/horizon/' @@ -507,7 +507,7 @@ 'keystone-base': { 'type': 'url', 'location': ('$tarballs_base/keystone/' - 'keystone-15.0.0.tar.gz')}, + 'keystone-15.0.1.tar.gz')}, 'kuryr-base': { 'type': 'url', 'location': ('$tarballs_base/kuryr/' @@ -572,7 +572,7 @@ 'neutron-base': { 'type': 'url', 'location': ('$tarballs_base/neutron/' - 'neutron-14.1.0.tar.gz')}, + 'neutron-14.2.0.tar.gz')}, 'neutron-base-plugin-neutron-fwaas': { 'type': 'url', 'location': ('$tarballs_base/neutron-fwaas/' @@ -648,11 +648,11 @@ 'neutron-server-ovn-plugin-networking-ovn': { 'type': 'url', 'location': ('$tarballs_base/networking-ovn/' - 'networking-ovn-6.0.1.tar.gz')}, + 'networking-ovn-6.1.0.tar.gz')}, 'neutron-metadata-agent-ovn-plugin-networking-ovn': { 'type': 'url', 'location': ('$tarballs_base/networking-ovn/' - 'networking-ovn-6.0.1.tar.gz')}, + 'networking-ovn-6.1.0.tar.gz')}, 'nova-base': { 'type': 'url', 'location': ('$tarballs_base/nova/' From d12137ce64fab7e8436aa1c2e32e3bc6ed47c1a1 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Wed, 17 Jun 2020 13:13:09 +0200 Subject: [PATCH 07/10] CI: Install python deps, mark oraclelinux rsyslog as skipped setuptools is missing from infra image - so it needs to be installed. Copy cat of a kolla-ansible change: https://review.opendev.org/#/c/735808/ Disables rsyslog build on oraclelinux (both source and binary) and tripleoclient (binary). Depends-On: https://review.opendev.org/737829 Change-Id: I6068abcbc0e73eeab77b0aa99983a95adc6defd3 --- kolla/image/build.py | 3 +++ tests/playbooks/pre.yml | 42 +++++++++++++++++++++++++++++++++++++++++ tests/playbooks/run.yml | 2 +- 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/kolla/image/build.py b/kolla/image/build.py index 5fd2cde0ce..7496573ea2 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -198,14 +198,17 @@ "monasca-thresh", "nova-mksproxy", "ovsdpdk", + "rsyslog", "searchlight-base", "solum-base", + "tripleoclient", "vmtp", "zun-base" ], 'oraclelinux+source': [ "bifrost-base", "ovsdpdk", + "rsyslog", "tripleoclient", # TODO(jeffrey4l): remove tripleo-ui when following bug is fixed # https://bugs.launchpad.net/tripleo/+bug/1744215 diff --git a/tests/playbooks/pre.yml b/tests/playbooks/pre.yml index b7f5a64912..ba546bfaff 100644 --- a/tests/playbooks/pre.yml +++ b/tests/playbooks/pre.yml @@ -16,7 +16,49 @@ path: "{{ kolla_build_logs_dir }}" state: directory + - block: + + - name: Ensure yum-utils is installed + # NOTE(mgoddard): The CentOS image used in CI has epel-release installed, + # but the configure-mirrors role used by Zuul disables epel. Since we + # install epel-release and expect epel to be enabled, enable it here. + package: + name: yum-utils + state: present + + - name: Enable the EPEL repository + command: yum-config-manager --enable epel + + become: true + when: + - ansible_os_family == "RedHat" + + - name: Install Python2 modules + become: true + package: + name: + - python-pip + - python-setuptools + - python-wheel + - python-virtualenv + + - name: Install virtualenv on Debian systems + # NOTE(hrw): On RedHat systems it is part of python3-virtualenv + package: + name: + - virtualenv + become: true + when: + ansible_os_family == "Debian" + + - name: Upgrade pip to latest version + # NOTE(mnasiadka): pip 8.x delivered with EPEL has problems installing + # zipp and configparser + become: true + command: "pip install --upgrade pip" + - name: Ensure tox is installed pip: name: tox + virtualenv: "{{ ansible_user_dir }}/tox-venv" become: true diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml index d22a388702..a26baa72e7 100644 --- a/tests/playbooks/run.yml +++ b/tests/playbooks/run.yml @@ -37,6 +37,6 @@ dest: /etc/kolla/template_overrides.j2 - name: Run tox - command: tox -e {{ action }}-{{ base_distro }}-{{ install_type }} + command: "{{ ansible_user_dir }}/tox-venv/bin/tox -e {{ action }}-{{ base_distro }}-{{ install_type }}" args: chdir: "{{ zuul.project.src_dir }}" From 6604041eb365952e269a518cd7a395d6f2b418c2 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Fri, 22 May 2020 15:54:52 +0200 Subject: [PATCH 08/10] CI: use wheels from infra mirror If they are available then we do not need to build them. Especially AArch64 jobs will speedup (once wheels are built). Change-Id: I79af6c37950e156018a9204fbcc7417cd7d41012 (adapted by hand) --- tests/templates/template_overrides.j2 | 1 + tests/vars/zuul.yml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/tests/templates/template_overrides.j2 b/tests/templates/template_overrides.j2 index 8997bc4ed9..9a36a78144 100644 --- a/tests/templates/template_overrides.j2 +++ b/tests/templates/template_overrides.j2 @@ -6,6 +6,7 @@ ENV PIP_INDEX_URL {{ nodepool_pypi_mirror }} ENV PIP_TRUSTED_HOST {{ nodepool_mirror_host }} +ENV PIP_EXTRA_INDEX_URL {{ nodepool_wheel_mirror }} RUN echo registry={{ nodepool_npmjs_proxy }} > /etc/npmrc \ && mkdir -p /usr/etc \ diff --git a/tests/vars/zuul.yml b/tests/vars/zuul.yml index e74e038f32..ca0454eea5 100644 --- a/tests/vars/zuul.yml +++ b/tests/vars/zuul.yml @@ -8,3 +8,6 @@ nodepool_grafana_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/grafana/" nodepool_cbs_centos_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/cbs.centos" nodepool_oraclelinux_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/oraclelinux/" nodepool_percona_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/percona" + +# NOTE(hrw): wheel cache goes over 80/443 not on 8080 +nodepool_wheel_mirror: "https://{{ zuul_site_mirror_fqdn }}/wheel/{{ ansible_distribution | lower }}-{{ ansible_distribution_version }}-{{ ansible_architecture | lower }}" From 37b3df14628de58f88f8c1869e8ba07dfe375c49 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 18 Jun 2020 11:53:05 +0100 Subject: [PATCH 09/10] Bump versions for Stein Change-Id: Id31883d93628958de61e8cd22de482c48b9c6c0b --- kolla/common/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kolla/common/config.py b/kolla/common/config.py index bcb5af4138..dfb7d06fbc 100755 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -331,7 +331,7 @@ 'cinder-base': { 'type': 'url', 'location': ('$tarballs_base/cinder/' - 'cinder-14.0.4.tar.gz')}, + 'cinder-14.1.0.tar.gz')}, 'congress-base': { 'type': 'url', 'location': ('$tarballs_base/congress/' @@ -672,7 +672,7 @@ 'octavia-base': { 'type': 'url', 'location': ('$tarballs_base/octavia/' - 'octavia-4.1.1.tar.gz')}, + 'octavia-4.1.2.tar.gz')}, 'panko-base': { 'type': 'url', 'location': ('$tarballs_base/panko/' From 7038e3ff03c6d08e7b794f9087675964ddbbed0a Mon Sep 17 00:00:00 2001 From: Michele Baldessari Date: Thu, 11 Jun 2020 11:47:02 +0200 Subject: [PATCH 10/10] Drop systemd support from nsswitch.conf on RHEL-based distros A bit like we did for I3e0e86026f5a4a78473bed824cd1682d3a020cd5 we should remove the nss-systemd lookup from containers. The reasons for this are as follows: 1) Just like for I3e0e86026f5a4a78473bed824cd1682d3a020cd5 when this nss module is triggered it tries to talk to dbus. It triggers a bunch of selinux denials and it makes little sense to open all containers to talk to dbus. In particular, if a container is run as non-privileged and bind-mounts /run from the host, we will hit selinux denials like the following: type=USER_AVC msg=audit(1592337775.860:74119): pid=1284 uid=81 auid=4294967295 ses=4294967295 subj=system_u:system_r:system_dbusd_t:s0-s0:c0.c1023 msg='avc: denied { send_msg } for msgtype=method_call interface=org.freedesktop.DBus member=Hello dest=org.freedesktop.DBus spid=406228 scontext=system_u:system_r:container_t:s0:c162,c886 tcontext=system_u:system_r:system_dbusd_t:s0-s0:c0.c1023 tclass=dbus permissive=0 exe="/usr/bin/dbus-daemon" sauid=81 hostname=? addr=? terminal=?'UID="dbus" AUID="unset" SAUID="dbus" 2) It just makes little sense in a kolla-world to have containers talk to dbus/systemd and it saves us some time when a lookup triggers the systemd module for whatever reason. Especially because the nss-systemd module does a few things which are not useful in a container (ensures that the root and nobody users and groups remain resolvable, SystemD's DynamicUser= feature, provide Lookup API via Varlink) The sed regex gives us the wanted results: $ diff -u /etc/nsswitch.conf.orig /etc/nsswitch.conf --- /etc/nsswitch.conf.orig 2020-06-19 07:18:10.974580755 +0000 +++ /etc/nsswitch.conf 2020-06-19 07:20:12.260230103 +0000 @@ -53,9 +53,9 @@ # group: db files # In order of likelihood of use to accelerate lookup. -passwd: sss files systemd +passwd: sss files shadow: files sss -group: sss files systemd +group: sss files hosts: files dns myhostname services: files sss netgroup: sss Related-Bug: #1883849 Change-Id: I81e5b7abf4571fece13a029e25911e9e4dece673 (cherry picked from commit dc2ddfa9750f94430a5b04b62fc1fa6adcb234f8) --- docker/base/Dockerfile.j2 | 3 ++- ...systemd-nss-on-rhel-based-distros-5d586fcdb9a82da7.yaml | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/disable-systemd-nss-on-rhel-based-distros-5d586fcdb9a82da7.yaml diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index 1f6356d1c0..13ed739eb0 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -404,7 +404,8 @@ RUN sed -i \ {% endif %} {% if base_distro == 'centos' or base_distro == 'rhel' %} -RUN sed -ri '/-session(\s+)optional(\s+)pam_systemd.so/d' /etc/pam.d/system-auth +RUN sed -ri '/-session(\s+)optional(\s+)pam_systemd.so/d' /etc/pam.d/system-auth \ + && sed -ri '/^[^#]/ s/systemd//g' /etc/nsswitch.conf {% endif %} COPY set_configs.py /usr/local/bin/kolla_set_configs diff --git a/releasenotes/notes/disable-systemd-nss-on-rhel-based-distros-5d586fcdb9a82da7.yaml b/releasenotes/notes/disable-systemd-nss-on-rhel-based-distros-5d586fcdb9a82da7.yaml new file mode 100644 index 0000000000..78521b9dcc --- /dev/null +++ b/releasenotes/notes/disable-systemd-nss-on-rhel-based-distros-5d586fcdb9a82da7.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Drop systemd support from nsswitch.conf on RHEL-based distros. This avoids + unneeded systemd nss lookups inside containers and it also avoids possible + selinux denials when a container bind mounts /run and makes the dbus socket + available inside the container only to be denied by selinux on the host.