diff --git a/.yamllint b/.yamllint index de74ae3a96..bfc9e04c14 100644 --- a/.yamllint +++ b/.yamllint @@ -1,3 +1,4 @@ +--- extends: default ignore: | .tox/ diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml index 2fc3dcf081..34a0a28479 100644 --- a/.zuul.d/base.yaml +++ b/.zuul.d/base.yaml @@ -40,8 +40,8 @@ - job: name: kolla-base parent: base - timeout: 10200 - post-timeout: 7200 + timeout: 10800 + post-timeout: 10800 pre-run: tests/playbooks/pre.yml run: tests/playbooks/run.yml post-run: tests/playbooks/post.yml diff --git a/.zuul.d/centos.yaml b/.zuul.d/centos.yaml index cc33602ef5..98a6d8ab50 100644 --- a/.zuul.d/centos.yaml +++ b/.zuul.d/centos.yaml @@ -8,7 +8,8 @@ - kolla-ansible-centos-binary - kolla-ansible-centos-source-upgrade - tripleo-build-containers-centos-7: - voting: true + # FIXME(yoctozepto): set to voting when TripleO CI is fixed + voting: false files: - ^.zuul.d/centos.yaml$ - ^docker/.*$ @@ -24,15 +25,6 @@ queue: kolla jobs: - kolla-build-centos-source - - tripleo-build-containers-centos-7: - voting: true - files: - - ^.zuul.d/centos.yaml$ - - ^docker/.*$ - - ^kolla/.*$ - - ^requirements.txt$ - - ^setup.cfg$ - - ^setup.py$ periodic: jobs: - kolla-publish-centos-source diff --git a/docker/aodh/aodh-api/Dockerfile.j2 b/docker/aodh/aodh-api/Dockerfile.j2 index abee6811ec..01ac126ad9 100644 --- a/docker/aodh/aodh-api/Dockerfile.j2 +++ b/docker/aodh/aodh-api/Dockerfile.j2 @@ -12,6 +12,10 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% set aodh_api_packages = ['aodh-api'] %} {% endif %} {{ macros.install_packages(aodh_api_packages | customizable("packages")) }} + + {% if base_package_type == 'deb' %} +RUN rm -rf /etc/apache2/sites-enabled/aodh-api.conf + {% endif %} {% endif %} COPY extend_start.sh /usr/local/bin/kolla_aodh_extend_start diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index 6bb5021115..ebe7a3ae2c 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -185,7 +185,7 @@ RUN {{ macros.install_packages( ['tar', 'yum-utils', 'https://dl.fedoraproject.o http://mirror.centos.org/centos-7/7/extras/x86_64/Packages/centos-release-nfs-ganesha28-1.0-2.el7.centos.noarch.rpm \ http://mirror.centos.org/centos-7/7/extras/x86_64/Packages/centos-release-ceph-nautilus-1.2-2.el7.centos.noarch.rpm \ http://mirror.centos.org/centos-7/7/extras/x86_64/Packages/centos-release-opstools-1-8.el7.noarch.rpm \ - http://mirror.centos.org/centos-7/7/extras/x86_64/Packages/centos-release-qemu-ev-1.0-3.el7.centos.noarch.rpm \ + http://mirror.centos.org/centos-7/7/extras/x86_64/Packages/centos-release-qemu-ev-1.0-4.el7.centos.noarch.rpm \ http://mirror.centos.org/centos-7/7/extras/x86_64/Packages/centos-release-virt-common-1-1.el7.centos.noarch.rpm \ http://mirror.centos.org/centos-7/7/extras/x86_64/Packages/centos-release-storage-common-2-2.el7.centos.noarch.rpm \ && sed -i 's/\$releasever/7/g' /etc/yum.repos.d/CentOS-*.repo \ @@ -353,7 +353,9 @@ COPY apt_preferences.{{ base_distro }} /etc/apt/preferences {% set remote_apt_keys = [ 'http://obs.linaro.org/ERP:/18.06/Debian_9/Release.key', 'https://download.docker.com/linux/debian/gpg', - 'https://packages.grafana.com/gpg.key' + 'https://packages.grafana.com/gpg.key', + 'http://buster-stein.debian.net/debian/dists/pubkey.gpg', + 'https://packages.treasuredata.com/GPG-KEY-td-agent', ] %} {% set base_apt_packages = base_apt_packages + ['sudo',] diff --git a/docker/base/nfs_ganesha.repo b/docker/base/nfs_ganesha.repo deleted file mode 100644 index ea0d0f07fc..0000000000 --- a/docker/base/nfs_ganesha.repo +++ /dev/null @@ -1,5 +0,0 @@ -[nfs_ganesha] -baseurl = http://download.ceph.com/nfs-ganesha/rpm-V2.5-stable/luminous/$basearch -gpgcheck = 1 -gpgkey = https://download.ceph.com/keys/release.asc -name = nfs-ganesha stable repo diff --git a/docker/base/sources.list.debian b/docker/base/sources.list.debian index 48e2e7137e..30b28d9053 100644 --- a/docker/base/sources.list.debian +++ b/docker/base/sources.list.debian @@ -11,3 +11,10 @@ deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main # main docker repo deb https://download.docker.com/linux/debian buster stable + +# Buster - Stein repos +deb http://buster-stein.debian.net/debian buster-stein-backports main +deb http://buster-stein.debian.net/debian buster-stein-backports-nochange main + +# td-agent for fluentd +deb http://packages.treasuredata.com/3/debian/stretch stretch contrib diff --git a/docker/ceph/ceph-mon/extend_start.sh b/docker/ceph/ceph-mon/extend_start.sh index af60faa177..188349788c 100644 --- a/docker/ceph/ceph-mon/extend_start.sh +++ b/docker/ceph/ceph-mon/extend_start.sh @@ -22,7 +22,7 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then # Generating initial keyrings and monmap ceph-authtool --create-keyring "${KEYRING_MON}" --gen-key -n mon. --cap mon 'allow *' - ceph-authtool --create-keyring "${KEYRING_ADMIN}" --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow' --cap mgr 'allow *' + ceph-authtool --create-keyring "${KEYRING_ADMIN}" --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' ceph-authtool --create-keyring "${KEYRING_RGW}" --gen-key -n client.radosgw.gateway --cap osd 'allow rwx' --cap mon 'allow rwx' ceph-authtool "${KEYRING_MON}" --import-keyring "${KEYRING_ADMIN}" ceph-authtool "${KEYRING_MON}" --import-keyring "${KEYRING_RGW}" diff --git a/docker/ceph/ceph-osd/extend_start.sh b/docker/ceph/ceph-osd/extend_start.sh index 31c4a0ffbc..f1a7f776f6 100644 --- a/docker/ceph/ceph-osd/extend_start.sh +++ b/docker/ceph/ceph-osd/extend_start.sh @@ -7,6 +7,42 @@ if [[ $(stat -c %a /var/log/kolla/ceph) != "755" ]]; then chmod 755 /var/log/kolla/ceph fi +# Inform the os about partition table changes +function partprobe_device { + local device=$1 + udevadm settle --timeout=600 + flock -s ${device} partprobe ${device} + udevadm settle --timeout=600 +} + +# In some cases, the disk partition will not appear immediately, so check every +# 1s, try up to 10 times. In general, this interval is enough. +function wait_partition_appear { + local dev_part=$1 + local part_name=$(echo ${dev_part} | awk -F '/' '{print $NF}') + for(( i=1; i<11; i++ )); do + flag=$(ls /dev | awk '/'"${part_name}"'/{print $0}' | wc -l) + if [[ "${flag}" -eq 0 ]]; then + echo "sleep 1 waits for the partition ${dev_part} to appear: ${i}" + sleep 1 + else + return 0 + fi + done + echo "The device /dev/${dev_part} does not appear within the limited time 10s." + exit 1 +} + +# Few storage device like loop or NVMe, wiil add "p" between disk & partition +# name if disk layout is end with number. This function will fix to correct format. +function part_name_checker { + if [[ $1 =~ .*[0-9] ]]; then + echo ${1}p${2} + else + echo ${1}${2} + fi +} + # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases # of the KOLLA_BOOTSTRAP variable being set, including empty. if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then @@ -22,45 +58,26 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then if [[ "${USE_EXTERNAL_JOURNAL}" == "False" ]]; then # Formatting disk for ceph if [[ "${OSD_STORETYPE}" == "bluestore" ]]; then - if [[ "${OSD_BS_DEV}" =~ "/dev/loop" ]]; then - sgdisk --zap-all -- "${OSD_BS_DEV}""p${OSD_BS_PARTNUM}" - else - sgdisk --zap-all -- "${OSD_BS_DEV}""${OSD_BS_PARTNUM}" - fi + sgdisk --zap-all -- "$(part_name_checker $OSD_BS_DEV $OSD_BS_PARTNUM)" if [ -n "${OSD_BS_BLK_DEV}" ] && [ "${OSD_BS_DEV}" != "${OSD_BS_BLK_DEV}" ] && [ -n "${OSD_BS_BLK_PARTNUM}" ]; then - if [[ "${OSD_BS_BLK_DEV}" =~ "/dev/loop" ]]; then - sgdisk --zap-all -- "${OSD_BS_BLK_DEV}""p${OSD_BS_BLK_PARTNUM}" - else - sgdisk --zap-all -- "${OSD_BS_BLK_DEV}""${OSD_BS_BLK_PARTNUM}" - fi + sgdisk --zap-all -- "$(part_name_checker ${OSD_BS_BLK_DEV} ${OSD_BS_BLK_PARTNUM})" else sgdisk --zap-all -- "${OSD_BS_DEV}" sgdisk --new=1:0:+100M --mbrtogpt -- "${OSD_BS_DEV}" sgdisk --largest-new=2 --mbrtogpt -- "${OSD_BS_DEV}" - partprobe || true + partprobe_device "${OSD_BS_DEV}" - if [[ "${OSD_BS_DEV}" =~ "/dev/loop" ]]; then - sgdisk --zap-all -- "${OSD_BS_DEV}"p2 - else - sgdisk --zap-all -- "${OSD_BS_DEV}"2 - fi + wait_partition_appear "$(part_name_checker $OSD_BS_DEV 2)" + sgdisk --zap-all -- "$(part_name_checker $OSD_BS_DEV 2)" fi if [ -n "${OSD_BS_WAL_DEV}" ] && [ "${OSD_BS_BLK_DEV}" != "${OSD_BS_WAL_DEV}" ] && [ -n "${OSD_BS_WAL_PARTNUM}" ]; then - if [[ "${OSD_BS_WAL_DEV}" =~ "/dev/loop" ]]; then - sgdisk --zap-all -- "${OSD_BS_WAL_DEV}""p${OSD_BS_WAL_PARTNUM}" - else - sgdisk --zap-all -- "${OSD_BS_WAL_DEV}""${OSD_BS_WAL_PARTNUM}" - fi + sgdisk --zap-all -- "$(part_name_checker $OSD_BS_WAL_DEV $OSD_BS_WAL_PARTNUM)" fi if [ -n "${OSD_BS_DB_DEV}" ] && [ "${OSD_BS_BLK_DEV}" != "${OSD_BS_DB_DEV}" ] && [ -n "${OSD_BS_DB_PARTNUM}" ]; then - if [[ "${OSD_BS_DB_DEV}" =~ "/dev/loop" ]]; then - sgdisk --zap-all -- "${OSD_BS_DB_DEV}""p${OSD_BS_DB_PARTNUM}" - else - sgdisk --zap-all -- "${OSD_BS_DB_DEV}""${OSD_BS_DB_PARTNUM}" - fi + sgdisk --zap-all -- "$(part_name_checker $OSD_BS_DB_DEV $OSD_BS_DB_PARTNUM)" fi else sgdisk --zap-all -- "${OSD_DEV}" @@ -78,13 +95,8 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}" mkdir -p "${OSD_DIR}" - if [[ "${OSD_BS_DEV}" =~ "/dev/loop" ]]; then - mkfs.xfs -f "${OSD_BS_DEV}""p${OSD_BS_PARTNUM}" - mount "${OSD_BS_DEV}""p${OSD_BS_PARTNUM}" "${OSD_DIR}" - else - mkfs.xfs -f "${OSD_BS_DEV}""${OSD_BS_PARTNUM}" - mount "${OSD_BS_DEV}""${OSD_BS_PARTNUM}" "${OSD_DIR}" - fi + mkfs.xfs -f "$(part_name_checker $OSD_BS_DEV $OSD_BS_PARTNUM)" + mount "$(part_name_checker $OSD_BS_DEV $OSD_BS_PARTNUM)" "${OSD_DIR}" # This will through an error about no key existing. That is normal. It then # creates the key in the next step. @@ -130,11 +142,8 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow profile osd' -i "${OSD_DIR}/keyring" - if [[ "${OSD_BS_DEV}" =~ "/dev/loop" ]]; then - umount "${OSD_BS_DEV}""p${OSD_BS_PARTNUM}" - else - umount "${OSD_BS_DEV}""${OSD_BS_PARTNUM}" - fi + umount "$(part_name_checker $OSD_BS_DEV $OSD_BS_PARTNUM)" + else OSD_ID=$(ceph osd create) OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}" diff --git a/docker/cyborg/cyborg-agent/Dockerfile.j2 b/docker/cyborg/cyborg-agent/Dockerfile.j2 index 398e6b2983..edd67e2b9d 100644 --- a/docker/cyborg/cyborg-agent/Dockerfile.j2 +++ b/docker/cyborg/cyborg-agent/Dockerfile.j2 @@ -4,6 +4,10 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% block cyborg_agent_header %}{% endblock %} {% import "macros.j2" as macros with context %} +{% set cyborg_agent_packages = [ + 'pciutils', +] %} + {% if install_type == 'binary' %} RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \ @@ -17,6 +21,8 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \ {% endif %} +{{ macros.install_packages(cyborg_agent_packages | customizable("packages")) }} + RUN {{ macros.install_pip(cyborg_agent_pip_packages | customizable("pip_packages")) }} {% if base_package_type == 'rpm' %} diff --git a/docker/elasticsearch/Dockerfile.j2 b/docker/elasticsearch/Dockerfile.j2 index f7deb92e91..7856444b9e 100644 --- a/docker/elasticsearch/Dockerfile.j2 +++ b/docker/elasticsearch/Dockerfile.j2 @@ -21,7 +21,7 @@ ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk/ 'elasticsearch', ] %} -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-{{debian_arch}}/ +ENV JAVA_HOME /usr/lib/jvm/java-{{java_version}}-openjdk-{{debian_arch}}/ {% endif %} diff --git a/docker/fluentd/Dockerfile.j2 b/docker/fluentd/Dockerfile.j2 index 1cf3821c3c..e4662ca8c1 100644 --- a/docker/fluentd/Dockerfile.j2 +++ b/docker/fluentd/Dockerfile.j2 @@ -21,7 +21,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'ruby-devel', 'rubygem-fluent-plugin-elasticsearch', 'rubygem-fluent-plugin-grok-parser', - 'rubygem-fluent-plugin-kubernetes_metadata_filter', 'rubygem-fluent-plugin-rewrite-tag-filter', 'rubygem-fluent-plugin-secure-forward' ] %} @@ -65,7 +64,6 @@ RUN chmod 755 /usr/local/bin/kolla_extend_start 'fluent-plugin-elasticsearch', 'fluent-plugin-grep', 'fluent-plugin-grok-parser:2.1.4', - 'fluent-plugin-kubernetes_metadata_filter', 'fluent-plugin-parser', 'fluent-plugin-rewrite-tag-filter:2.0.0', 'fluent-plugin-secure-forward', @@ -75,7 +73,7 @@ RUN chmod 755 /usr/local/bin/kolla_extend_start {{ macros.install_fluent_plugins(fluentd_plugins | customizable("plugins")) }} # Build and install Fluentd output plugin for Monasca Log API -ARG monasca_output_plugin_tag=0.1.0 +ARG monasca_output_plugin_tag=0.1.1 ARG monasca_output_plugin_url=https://github.com/monasca/fluentd-monasca/archive/$monasca_output_plugin_tag.tar.gz ADD $monasca_output_plugin_url /tmp/fluentd-monasca.tar.gz RUN tar -xvf /tmp/fluentd-monasca.tar.gz -C /tmp \ diff --git a/docker/gnocchi/gnocchi-api/Dockerfile.j2 b/docker/gnocchi/gnocchi-api/Dockerfile.j2 index 863018336f..535831cffd 100644 --- a/docker/gnocchi/gnocchi-api/Dockerfile.j2 +++ b/docker/gnocchi/gnocchi-api/Dockerfile.j2 @@ -15,6 +15,10 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% set gnocchi_api_packages = ['gnocchi-api'] %} {% endif %} {{ macros.install_packages(gnocchi_api_packages | customizable("packages")) }} + + {% if base_package_type == 'deb' %} +RUN rm -rf /etc/apache2/sites-enabled/gnocchi-api.conf + {% endif %} {% endif %} COPY extend_start.sh /usr/local/bin/kolla_gnocchi_extend_start diff --git a/docker/horizon/Dockerfile.j2 b/docker/horizon/Dockerfile.j2 index cda759521b..4c3ba13305 100644 --- a/docker/horizon/Dockerfile.j2 +++ b/docker/horizon/Dockerfile.j2 @@ -67,6 +67,7 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \ 'python3-designate-dashboard', 'python3-heat-dashboard', 'python3-manila-ui', + 'python3-octavia-dashboard', 'python3-sahara-dashboard', 'python3-trove-dashboard', 'tzdata' @@ -83,6 +84,8 @@ RUN echo > /etc/apache2/ports.conf \ && cp /usr/share/openstack-dashboard/openstack_dashboard/conf/*.json /etc/openstack-dashboard \ && cp /usr/share/openstack-dashboard/manage.py /usr/bin/manage.py \ && rm /etc/apache2/conf-enabled/openstack-dashboard.conf \ + && rm /etc/openstack-dashboard/local_settings.py \ + && ln -s /etc/openstack-dashboard/local_settings /etc/openstack-dashboard/local_settings.py \ && for locale in /usr/lib/python{{distro_python_version}}/site-packages/*/locale; do \ (cd ${locale%/*} && /usr/bin/python3 /usr/bin/manage.py compilemessages) \ done diff --git a/docker/kafka/Dockerfile.j2 b/docker/kafka/Dockerfile.j2 index 8a7ec3fd09..bcb79d8a88 100644 --- a/docker/kafka/Dockerfile.j2 +++ b/docker/kafka/Dockerfile.j2 @@ -12,8 +12,15 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'java-1.8.0-openjdk-headless', ] %} {% elif base_package_type == 'deb' %} + + {% if base_distro == 'debian' %} + {% set java_version = '11' %} + {% elif base_distro == 'ubuntu' %} + {% set java_version = '8' %} + {% endif %} + {% set kafka_packages = [ - 'openjdk-8-jre-headless', + 'openjdk-' + java_version + '-jre-headless', ] %} {% endif %} diff --git a/docker/logstash/Dockerfile.j2 b/docker/logstash/Dockerfile.j2 index 36d62676ee..9652e8ab65 100644 --- a/docker/logstash/Dockerfile.j2 +++ b/docker/logstash/Dockerfile.j2 @@ -15,12 +15,19 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk/ {% elif base_package_type == 'deb' %} + + {% if base_distro == 'debian' %} + {% set java_version = '11' %} + {% elif base_distro == 'ubuntu' %} + {% set java_version = '8' %} + {% endif %} + {% set logstash_packages = [ - 'openjdk-8-jre-headless', + 'openjdk-' + java_version + '-jre-headless', 'logrotate' ] %} -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-{{debian_arch}}/ +ENV JAVA_HOME /usr/lib/jvm/java-{{java_version}}-openjdk-{{debian_arch}}/ {% endif %} diff --git a/docker/monasca/monasca-agent/Dockerfile.j2 b/docker/monasca/monasca-agent/Dockerfile.j2 index 096e22c022..b56982a02b 100644 --- a/docker/monasca/monasca-agent/Dockerfile.j2 +++ b/docker/monasca/monasca-agent/Dockerfile.j2 @@ -15,7 +15,8 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \ ADD monasca-agent-archive /monasca-agent-source {% set monasca_agent_pip_packages = [ - '/monasca-agent' + '/monasca-agent', + 'prometheus_client' ] %} RUN ln -s monasca-agent-source/* monasca-agent \ diff --git a/docker/monasca/monasca-grafana/Dockerfile.j2 b/docker/monasca/monasca-grafana/Dockerfile.j2 index a43d71dc03..3e37a16878 100644 --- a/docker/monasca/monasca-grafana/Dockerfile.j2 +++ b/docker/monasca/monasca-grafana/Dockerfile.j2 @@ -48,7 +48,9 @@ ARG monasca_grafana_version=grafana4 ARG monasca_grafana_url=https://github.com/monasca/grafana/archive/$monasca_grafana_version.tar.gz {% block monasca_grafana_install %} -RUN gem install rake fpm \ +# NOTE(mgoddard): Pinning rake because rake 13.0.0 depends on Ruby 2.2, and CentOS 7 only provides +# Ruby 2.0. +RUN gem install rake:"~>12" fpm \ && curl -sSL -o /tmp/monasca-grafana.tgz ${monasca_grafana_url} \ && mkdir -p ${monasca_grafana_build_path} \ && tar --strip 1 -xvf /tmp/monasca-grafana.tgz -C ${monasca_grafana_build_path} \ diff --git a/docker/monasca/monasca-thresh/Dockerfile.j2 b/docker/monasca/monasca-thresh/Dockerfile.j2 index 6897996002..f76ef50e0d 100644 --- a/docker/monasca/monasca-thresh/Dockerfile.j2 +++ b/docker/monasca/monasca-thresh/Dockerfile.j2 @@ -24,12 +24,19 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \ ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk/ {% elif base_package_type == 'deb' %} + + {% if base_distro == 'debian' %} + {% set java_version = '11' %} + {% elif base_distro == 'ubuntu' %} + {% set java_version = '8' %} + {% endif %} + {% set monasca_thresh_packages = [ - 'openjdk-8-jdk-headless', + 'openjdk-' + java_version + '-jdk-headless', 'maven', ] %} -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-{{debian_arch}}/ +ENV JAVA_HOME /usr/lib/jvm/java-{{java_version}}-openjdk-{{debian_arch}}/ {% endif %} diff --git a/docker/monasca/monasca-thresh/extend_start.sh b/docker/monasca/monasca-thresh/extend_start.sh index 77e835d447..6c4dda35d5 100644 --- a/docker/monasca/monasca-thresh/extend_start.sh +++ b/docker/monasca/monasca-thresh/extend_start.sh @@ -1,15 +1,44 @@ #!/bin/bash -# Create log directory, with appropriate permissions +# Create log and data directories, with appropriate permissions MONASCA_LOG_DIR="/var/log/kolla/monasca" +MONASCA_DATA_DIR="/var/lib/monasca-thresh/data" +MONASCA_WORKER_DIR="/var/lib/monasca-thresh/worker-artifacts" if [[ ! -d "$MONASCA_LOG_DIR" ]]; then mkdir -p $MONASCA_LOG_DIR fi +if [[ ! -d "$MONASCA_DATA_DIR" ]]; then + mkdir -p $MONASCA_DATA_DIR +fi +if [[ ! -d "$MONASCA_WORKER_DIR" ]]; then + mkdir -p $MONASCA_WORKER_DIR +fi if [[ $(stat -c %U:%G ${MONASCA_LOG_DIR}) != "monasca:kolla" ]]; then chown monasca:kolla ${MONASCA_LOG_DIR} fi +if [[ $(stat -c %U:%G ${MONASCA_DATA_DIR}) != "monasca:kolla" ]]; then + chown monasca:kolla ${MONASCA_DATA_DIR} +fi +if [[ $(stat -c %U:%G ${MONASCA_WORKER_DIR}) != "monasca:kolla" ]]; then + chown monasca:kolla ${MONASCA_WORKER_DIR} +fi if [[ $(stat -c %a ${MONASCA_LOG_DIR}) != "755" ]]; then chmod 755 ${MONASCA_LOG_DIR} fi +if [[ $(stat -c %a ${MONASCA_DATA_DIR}) != "755" ]]; then + chmod 755 ${MONASCA_DATA_DIR} +fi +if [[ $(stat -c %a ${MONASCA_WORKER_DIR}) != "755" ]]; then + chmod 755 ${MONASCA_WORKER_DIR} +fi + +# Delete the contents of data and worker-artifacts directories as +# Apache Storm doesn't clear temp files unless shutdown gracefully. +if [[ $(ls -Ab ${MONASCA_DATA_DIR}) != "" ]]; then + find ${MONASCA_DATA_DIR} -mindepth 1 -delete +fi +if [[ $(ls -Ab ${MONASCA_WORKER_DIR}) != "" ]]; then + find ${MONASCA_WORKER_DIR} -mindepth 1 -delete +fi . /usr/local/bin/kolla_monasca_extend_start diff --git a/docker/nova/nova-libvirt/Dockerfile.j2 b/docker/nova/nova-libvirt/Dockerfile.j2 index 82aaf4b4f8..fb75e443a8 100644 --- a/docker/nova/nova-libvirt/Dockerfile.j2 +++ b/docker/nova/nova-libvirt/Dockerfile.j2 @@ -44,7 +44,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'python3-rados', 'python3-rbd', 'qemu-block-extra', - 'qemu-kvm', 'qemu-system', 'trousers' ] %} @@ -53,6 +52,10 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% set nova_libvirt_packages = nova_libvirt_packages + [ 'qemu-efi' ] %} + {% elif base_arch == "x86_64" %} + {% set nova_libvirt_packages = nova_libvirt_packages + [ + 'qemu-kvm' + ] %} {% endif %} {% if base_arch != "ppc64le" %} diff --git a/docker/octavia/octavia-api/Dockerfile.j2 b/docker/octavia/octavia-api/Dockerfile.j2 index cebdd2e547..8fa03cd91b 100644 --- a/docker/octavia/octavia-api/Dockerfile.j2 +++ b/docker/octavia/octavia-api/Dockerfile.j2 @@ -22,8 +22,11 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build ] %} {% endif %} {% elif base_package_type == 'deb' %} -RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \ - && /bin/false + {% set octavia_api_packages = [ + 'octavia-api', + 'apache2', + 'libapache2-mod-wsgi-py3' + ] %} {% endif %} diff --git a/docker/octavia/octavia-base/Dockerfile.j2 b/docker/octavia/octavia-base/Dockerfile.j2 index 0efcc6db77..526965f511 100644 --- a/docker/octavia/octavia-base/Dockerfile.j2 +++ b/docker/octavia/octavia-base/Dockerfile.j2 @@ -13,8 +13,9 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'openstack-octavia-common' ] %} {% elif base_package_type == 'deb' %} -RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \ - && /bin/false + {% set octavia_base_packages = [ + 'octavia-common' + ] %} {% endif %} {{ macros.install_packages(octavia_base_packages | customizable("packages")) }} diff --git a/docker/octavia/octavia-health-manager/Dockerfile.j2 b/docker/octavia/octavia-health-manager/Dockerfile.j2 index 4dfcbe189c..6983f793cf 100644 --- a/docker/octavia/octavia-health-manager/Dockerfile.j2 +++ b/docker/octavia/octavia-health-manager/Dockerfile.j2 @@ -11,8 +11,9 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'openstack-octavia-health-manager' ] %} {% elif base_package_type == 'deb' %} -RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \ - && /bin/false + {% set octavia_health_manager_packages = [ + 'octavia-health-manager' + ] %} {% endif %} {{ macros.install_packages(octavia_health_manager_packages | customizable("packages")) }} diff --git a/docker/octavia/octavia-housekeeping/Dockerfile.j2 b/docker/octavia/octavia-housekeeping/Dockerfile.j2 index c7a25cf7af..31688d6fed 100644 --- a/docker/octavia/octavia-housekeeping/Dockerfile.j2 +++ b/docker/octavia/octavia-housekeeping/Dockerfile.j2 @@ -11,8 +11,9 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'openstack-octavia-housekeeping' ] %} {% elif base_package_type == 'deb' %} -RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \ - && /bin/false + {% set octavia_housekeeping_packages = [ + 'octavia-housekeeping' + ] %} {% endif %} {{ macros.install_packages(octavia_housekeeping_packages | customizable("packages")) }} diff --git a/docker/octavia/octavia-worker/Dockerfile.j2 b/docker/octavia/octavia-worker/Dockerfile.j2 index 2968524957..40db1bc80d 100644 --- a/docker/octavia/octavia-worker/Dockerfile.j2 +++ b/docker/octavia/octavia-worker/Dockerfile.j2 @@ -11,8 +11,9 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'openstack-octavia-worker' ] %} {% elif base_package_type == 'deb' %} -RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \ - && /bin/false + {% set octavia_worker_packages = [ + 'octavia-worker' + ] %} {% endif %} {{ macros.install_packages(octavia_worker_packages | customizable("packages")) }} diff --git a/docker/opendaylight/Dockerfile.j2 b/docker/opendaylight/Dockerfile.j2 index d71f586422..9e2e24b453 100644 --- a/docker/opendaylight/Dockerfile.j2 +++ b/docker/opendaylight/Dockerfile.j2 @@ -15,10 +15,18 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'opendaylight', ] %} {% elif base_package_type == 'deb' %} + + {% if base_distro == 'debian' %} + {% set java_version = '11' %} + {% elif base_distro == 'ubuntu' %} + {% set java_version = '8' %} + {% endif %} + {% set opendaylight_packages = [ 'opendaylight', - 'openjdk-8-jre-headless', + 'openjdk-' + java_version + '-jre-headless', ] %} + # NOTE(egonzalez): ODL fails to install in debian family images with # existing odl user. First install odl and then allow usage of # install_packages macro for custom configs. diff --git a/docker/openvswitch/openvswitch-db-server/start_ovsdb_server.sh b/docker/openvswitch/openvswitch-db-server/start_ovsdb_server.sh index 5c92fba781..bb8bba65e1 100755 --- a/docker/openvswitch/openvswitch-db-server/start_ovsdb_server.sh +++ b/docker/openvswitch/openvswitch-db-server/start_ovsdb_server.sh @@ -26,5 +26,5 @@ if [ ! -e $ovs_bridge ] && [ ! -e $ovs_ext_intf ]; then /usr/sbin/ovsdb-server /var/lib/openvswitch/conf.db -vconsole:emer -vsyslog:err -vfile:info --remote=punix:/var/run/openvswitch/db.sock --remote=ptcp:6640 --log-file=/var/log/kolla/openvswitch/ovsdb-server.log else # NOTE: (sbezverk) This part is executed only by kolla-ansible deployment. - /usr/sbin/ovsdb-server /var/lib/openvswitch/conf.db -vconsole:emer -vsyslog:err -vfile:info --remote=punix:/run/openvswitch/db.sock --remote=ptcp:6640:$ovsdb_ip --log-file=/var/log/kolla/openvswitch/ovsdb-server.log + /usr/sbin/ovsdb-server /var/lib/openvswitch/conf.db -vconsole:emer -vsyslog:err -vfile:info --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --remote=punix:/run/openvswitch/db.sock --remote=ptcp:6640:$ovsdb_ip --log-file=/var/log/kolla/openvswitch/ovsdb-server.log fi diff --git a/docker/placement/placement-base/Dockerfile.j2 b/docker/placement/placement-base/Dockerfile.j2 index 6c82e7e283..5dd6520d7a 100644 --- a/docker/placement/placement-base/Dockerfile.j2 +++ b/docker/placement/placement-base/Dockerfile.j2 @@ -25,8 +25,14 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build ] %} {% endif %} + {% if base_distro in ['debian'] %} + {% set mysql_migrate_path = '/usr/share/placement-common/mysql-migrate-db.sh' %} + {% else %} + {% set mysql_migrate_path = '/usr/share/placement/mysql-migrate-db.sh' %} + {% endif %} + {{ macros.install_packages(placement_base_packages | customizable("packages")) }} \ - && cp /usr/share/placement/mysql-migrate-db.sh /opt/ \ + && cp {{ mysql_migrate_path }} /opt/ \ && chmod 755 /opt/mysql-migrate-db.sh {% elif install_type == 'source' %} diff --git a/docker/prometheus/prometheus-openstack-exporter/Dockerfile.j2 b/docker/prometheus/prometheus-openstack-exporter/Dockerfile.j2 index 40282ab06b..ff5d55e130 100644 --- a/docker/prometheus/prometheus-openstack-exporter/Dockerfile.j2 +++ b/docker/prometheus/prometheus-openstack-exporter/Dockerfile.j2 @@ -4,11 +4,11 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% block prometheus_openstack_exporter_header %}{% endblock %} {% block prometheus_openstack_exporter_repository_version %} -ENV prometheus_openstack_exporter_version=0.0.7 +ENV prometheus_openstack_exporter_version=0.2.1 {% endblock %} {% block prometheus_openstack_exporter_install %} -RUN curl -sSL -o /tmp/prometheus_openstack_exporter.tar.gz https://github.com/Linaro/openstack-exporter/releases/download/v${prometheus_openstack_exporter_version}/openstack-exporter-${prometheus_openstack_exporter_version}.linux-{{debian_arch}}.tar.gz \ +RUN curl -sSL -o /tmp/prometheus_openstack_exporter.tar.gz https://github.com/openstack-exporter/openstack-exporter/releases/download/v${prometheus_openstack_exporter_version}/openstack-exporter-${prometheus_openstack_exporter_version}.linux-{{debian_arch}}.tar.gz \ && tar xvf /tmp/prometheus_openstack_exporter.tar.gz -C /opt/ \ && rm -f /tmp/prometheus_openstack_exporter.tar.gz \ && ln -s /opt/openstack-exporter* /opt/openstack-exporter diff --git a/docker/sensu/sensu-client/Dockerfile.j2 b/docker/sensu/sensu-client/Dockerfile.j2 index 28779ed4f8..51913b340e 100644 --- a/docker/sensu/sensu-client/Dockerfile.j2 +++ b/docker/sensu/sensu-client/Dockerfile.j2 @@ -88,7 +88,9 @@ RUN echo '{{ image_name }} not yet available for {{ base_distro }}' \ # TODO(mandre) Use packaged sensu plugins from centos-opstools for binary distro # http://cbs.centos.org/koji/search?match=glob&type=package&terms=*sensu* -RUN sensu-install --plugins {{ sensu_plugins | customizable('plugins') | join (',') }} +# NOTE(hrw): whois 5.0.0 requires Ruby 2.4+ while CentOS has 2.0 +# NOTE(yoctozepto): pinning minitest for the same reason +RUN {%if base_package_type == 'rpm' %} gem install whois:"<5" minitest:"~>5.11.3" && {% endif %} sensu-install --plugins {{ sensu_plugins | customizable('plugins') | join (',') }} {% endblock %} {% block sensu_client_footer %}{% endblock %} diff --git a/docker/storm/Dockerfile.j2 b/docker/storm/Dockerfile.j2 index aa3c13f5e6..d3b3f8acf5 100644 --- a/docker/storm/Dockerfile.j2 +++ b/docker/storm/Dockerfile.j2 @@ -12,8 +12,15 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'java-1.8.0-openjdk-headless', ] %} {% elif base_package_type == 'deb' %} + + {% if base_distro == 'debian' %} + {% set java_version = '11' %} + {% elif base_distro == 'ubuntu' %} + {% set java_version = '8' %} + {% endif %} + {% set storm_packages = [ - 'openjdk-8-jre-headless', + 'openjdk-' + java_version + '-jre-headless', ] %} {% endif %} @@ -21,7 +28,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% block storm_version %} ENV storm_version=1.2.2 -ENV storm_url=http://www.mirrorservice.org/sites/ftp.apache.org/storm/apache-storm-${storm_version}/apache-storm-${storm_version}.tar.gz +ENV storm_url=https://archive.apache.org/dist/storm/apache-storm-${storm_version}/apache-storm-${storm_version}.tar.gz ENV storm_pkg_sha512sum=0a1120b8df7b22edc75f0a412d625841f72f3fb8e9ff5d413d510908d68ea1f0c17d68c1a7f1eda427b40902452e9efcae902c36499b558592e41cc1079de2e0 {% endblock %} diff --git a/docker/swift/swift-base/Dockerfile.j2 b/docker/swift/swift-base/Dockerfile.j2 index c5b81d6570..fbf4b3556f 100644 --- a/docker/swift/swift-base/Dockerfile.j2 +++ b/docker/swift/swift-base/Dockerfile.j2 @@ -10,24 +10,30 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% if install_type == 'binary' %} {% if base_package_type == 'rpm' %} {% set swift_base_packages = [ - 'openstack-swift', - 'nmap-ncat' - ] - %} + 'nmap-ncat', + 'openstack-swift', + ] %} {% elif base_package_type == 'deb' %} {% set swift_base_packages = [ - 'swift', - 'netcat-openbsd' - ] - %} + 'netcat-openbsd', + 'rsync', + 'swift', + ] %} {% endif %} {{ macros.install_packages(swift_base_packages | customizable("packages")) }} {% elif install_type == 'source' %} {% if base_package_type == 'rpm' %} - {% set swift_base_packages = ['liberasurecode-devel','nmap-ncat'] %} + {% set swift_base_packages = [ + 'liberasurecode-devel', + 'nmap-ncat', + ] %} {% elif base_package_type == 'deb' %} - {% set swift_base_packages = ['liberasurecode-dev','netcat-openbsd'] %} + {% set swift_base_packages = [ + 'liberasurecode-dev', + 'netcat-openbsd', + 'rsync', + ] %} {% endif %} {{ macros.install_packages(swift_base_packages | customizable("packages")) }} diff --git a/docker/tempest/Dockerfile.j2 b/docker/tempest/Dockerfile.j2 index ee8d938d51..ec0edf161d 100644 --- a/docker/tempest/Dockerfile.j2 +++ b/docker/tempest/Dockerfile.j2 @@ -9,7 +9,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% if install_type == 'binary' %} {% if base_package_type == 'rpm' %} - {% set tempest_packages = ['openstack-tempest'] %} + {% set tempest_packages = ['openstack-tempest-all'] %} {% elif base_package_type == 'deb' %} {% set tempest_packages = ['tempest'] %} diff --git a/docker/zun/zun-compute/Dockerfile.j2 b/docker/zun/zun-compute/Dockerfile.j2 index 33061893b3..0d8d385c86 100644 --- a/docker/zun/zun-compute/Dockerfile.j2 +++ b/docker/zun/zun-compute/Dockerfile.j2 @@ -3,6 +3,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% block zun_compute_header %}{% endblock %} +{% import "macros.j2" as macros with context %} + {% if install_type == 'binary' %} RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \ @@ -10,6 +12,12 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \ {% endif %} +{% set zun_compute_packages = [ + 'pciutils', +] %} + +{{ macros.install_packages(zun_compute_packages | customizable("packages")) }} + COPY zun_sudoers /etc/sudoers.d/kolla_zun_sudoers COPY extend_start.sh /usr/local/bin/kolla_zun_extend_start diff --git a/kolla/common/config.py b/kolla/common/config.py index 4122ebc783..a1e000c251 100755 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -87,6 +87,7 @@ 'keystone', 'neutron', 'nova-', + 'placement', 'swift', ], help='Main images'), @@ -146,6 +147,7 @@ 'memcached', 'neutron', 'nova-', + 'placement', 'openvswitch', 'rabbitmq', ], @@ -165,6 +167,7 @@ 'memcached', 'neutron', 'nova-', + 'placement', 'openvswitch', 'rabbitmq', ], @@ -328,7 +331,7 @@ 'cinder-base': { 'type': 'url', 'location': ('$tarballs_base/cinder/' - 'cinder-14.0.0.tar.gz')}, + 'cinder-14.0.2.tar.gz')}, 'congress-base': { 'type': 'url', 'location': ('$tarballs_base/congress/' @@ -488,15 +491,15 @@ 'horizon-plugin-zun-ui': { 'type': 'url', 'location': ('$tarballs_base/zun-ui/' - 'zun-ui-3.0.0.tar.gz')}, + 'zun-ui-3.0.1.tar.gz')}, 'ironic-base': { 'type': 'url', 'location': ('$tarballs_base/ironic/' - 'ironic-12.1.1.tar.gz')}, + 'ironic-12.1.2.tar.gz')}, 'ironic-inspector': { 'type': 'url', 'location': ('$tarballs_base/ironic-inspector/' - 'ironic-inspector-8.2.2.tar.gz')}, + 'ironic-inspector-8.2.3.tar.gz')}, 'karbor-base': { 'type': 'url', 'location': ('$tarballs_base/karbor/' @@ -516,11 +519,11 @@ 'magnum-base': { 'type': 'url', 'location': ('$tarballs_base/magnum/' - 'magnum-8.0.0.tar.gz')}, + 'magnum-8.1.0.tar.gz')}, 'manila-base': { 'type': 'url', 'location': ('$tarballs_base/manila/' - 'manila-8.0.0.tar.gz')}, + 'manila-8.0.1.tar.gz')}, 'mistral-base': { 'type': 'url', 'location': ('$tarballs_base/mistral/' @@ -653,7 +656,7 @@ 'nova-base': { 'type': 'url', 'location': ('$tarballs_base/nova/' - 'nova-19.0.1.tar.gz')}, + 'nova-19.0.2.tar.gz')}, 'nova-base-plugin-blazar': { 'type': 'url', 'location': ('$tarballs_base/blazar-nova/' @@ -669,7 +672,7 @@ 'octavia-base': { 'type': 'url', 'location': ('$tarballs_base/octavia/' - 'octavia-4.0.1.tar.gz')}, + 'octavia-4.1.0.tar.gz')}, 'panko-base': { 'type': 'url', 'location': ('$tarballs_base/panko/' @@ -677,7 +680,7 @@ 'placement-base': { 'type': 'url', 'location': ('$tarballs_base/placement/' - 'openstack-placement-1.0.0.tar.gz')}, + 'openstack-placement-1.1.0.tar.gz')}, 'tempest-plugin-tempest-conf': { 'type': 'url', 'location': ('$tarballs_base/python-tempestconf/' @@ -721,7 +724,7 @@ 'tempest-plugin-manila': { 'type': 'url', 'location': ('$tarballs_base/manila-tempest-plugin/' - 'manila-tempest-plugin-0.2.0.tar.gz')}, + 'manila-tempest-plugin-0.3.0.tar.gz')}, 'tempest-plugin-mistral': { 'type': 'url', 'location': ('$tarballs_base/mistral-tempest-plugin/' diff --git a/kolla/image/build.py b/kolla/image/build.py index 8e7b546e37..2e88061049 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -123,7 +123,6 @@ "monasca-thresh", "nova-mksproxy", "novajoin-base", - "octavia-base", # There is no qdrouterd package for ubuntu bionic "qdrouterd", "searchlight-base", @@ -368,6 +367,9 @@ def push_image(self, image): image.status = STATUS_ERROR self.logger.error(response['errorDetail']['message']) + # Reset any previous errors. + image.status = STATUS_BUILT + class BuildTask(DockerTask): """Task that builds out an image.""" diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py index 1fe0f27d31..b0ce009a9a 100644 --- a/kolla/tests/test_build.py +++ b/kolla/tests/test_build.py @@ -77,6 +77,40 @@ def test_push_image(self, mock_client): pusher.run() mock_client().push.assert_called_once_with( self.image.canonical_name, decode=True, stream=True) + self.assertTrue(pusher.success) + + @mock.patch('docker.version', '3.0.0') + @mock.patch.dict(os.environ, clear=True) + @mock.patch('docker.APIClient') + def test_push_image_failure(self, mock_client): + self.dc = mock_client + mock_client().push.side_effect = Exception + pusher = build.PushTask(self.conf, self.image) + pusher.run() + mock_client().push.assert_called_once_with( + self.image.canonical_name, decode=True, stream=True) + self.assertFalse(pusher.success) + self.assertEqual(build.STATUS_PUSH_ERROR, self.image.status) + + @mock.patch('docker.version', '3.0.0') + @mock.patch.dict(os.environ, clear=True) + @mock.patch('docker.APIClient') + def test_push_image_failure_retry(self, mock_client): + self.dc = mock_client + mock_client().push.side_effect = [Exception, []] + pusher = build.PushTask(self.conf, self.image) + pusher.run() + mock_client().push.assert_called_once_with( + self.image.canonical_name, decode=True, stream=True) + self.assertFalse(pusher.success) + self.assertEqual(build.STATUS_PUSH_ERROR, self.image.status) + + # Try again, this time without exception. + pusher.reset() + pusher.run() + self.assertEqual(2, mock_client().push.call_count) + self.assertTrue(pusher.success) + self.assertEqual(build.STATUS_BUILT, self.image.status) @mock.patch.dict(os.environ, clear=True) @mock.patch('docker.APIClient')