From b25cecfb04cea14d17d419c6be49b84dcb76a2ff Mon Sep 17 00:00:00 2001 From: vivostar Date: Thu, 2 Jun 2022 21:26:32 +0800 Subject: [PATCH] BIGTOP-3691. Bump Puppet to Puppet7 --- bigtop-deploy/puppet/hiera.yaml | 27 +++++++--- bigtop-deploy/puppet/manifests/bigtop_repo.pp | 15 ++---- bigtop-deploy/puppet/manifests/jdk.pp | 2 +- bigtop_toolchain/bin/puppetize.sh | 51 ++++++++++++++----- bigtop_toolchain/manifests/puppet_modules.pp | 15 ++---- docker/bigtop-puppet/build.sh | 6 +-- docker/bigtop-slaves/Dockerfile.template | 2 +- docker/bigtop-slaves/build.sh | 26 ++-------- provisioner/docker/docker-compose.yml | 4 +- provisioner/docker/docker-hadoop.sh | 6 +-- 10 files changed, 78 insertions(+), 76 deletions(-) diff --git a/bigtop-deploy/puppet/hiera.yaml b/bigtop-deploy/puppet/hiera.yaml index 848c550377..6b92356101 100644 --- a/bigtop-deploy/puppet/hiera.yaml +++ b/bigtop-deploy/puppet/hiera.yaml @@ -1,8 +1,21 @@ --- -:yaml: - :datadir: /etc/puppet/hieradata -:hierarchy: - - site - - "bigtop/%{hadoop_hiera_ha_path}" - - bigtop/cluster - - bigtop/repo +version: 5 +defaults: + datadir: /etc/puppetlabs/code/environments/production/hieradata + data_hash: yaml_data +hierarchy: +# the site path can be dynamic. here is an example +# - name: "my node site path" +# path: "%{::fqdn}_site.yaml" +# and in the hieradata dir, there should be such as +# bigtopc01.vm_site.yaml +# bigtopc02.vm_site.yaml + + - name: "site hiera data path" + path: "site.yaml" +# - name: "hadoop hiera ha path" +# path: "bigtop/%{hadoop_hiera_ha_path}.yaml" + - name: "bigtop cluster hiera data path" + path: "bigtop/cluster.yaml" + - name: "bigtop repo hiera data path" + path: "bigtop/repo.yaml" \ No newline at end of file diff --git a/bigtop-deploy/puppet/manifests/bigtop_repo.pp b/bigtop-deploy/puppet/manifests/bigtop_repo.pp index 1aa767f674..5a9f71a759 100644 --- a/bigtop-deploy/puppet/manifests/bigtop_repo.pp +++ b/bigtop-deploy/puppet/manifests/bigtop_repo.pp @@ -20,7 +20,7 @@ $default_repo = "http://repos.bigtop.apache.org/releases/${bigtop_repo_default_version}/${lower_os}/${operatingsystemmajrelease}/${architecture}" case $::operatingsystem { - /(OracleLinux|Amazon|CentOS|Fedora|RedHat)/: { + /(OracleLinux|Amazon|CentOS|Fedora|RedHat|Rocky)/: { $baseurls_array = any2array(hiera("bigtop::bigtop_repo_uri", $default_repo)) each($baseurls_array) |$count, $baseurl| { notify { "Baseurl: $baseurl": } @@ -98,17 +98,8 @@ } } - # BIGTOP-3580. After the Debian 11 release, `apt-get update` fails on Debian 10 - # if its repository info is obsolete. In such case, the repository info - # should be updated by the `--allow-releaseinfo-change` option. - if ($operatingsystem == 'Debian' and 0 <= versioncmp($operatingsystemrelease, "10")) { - exec { 'bigtop-apt-update': - command => '/usr/bin/apt-get update --allow-releaseinfo-change' - } - Apt::Conf<||> -> Apt::Key<||> -> Exec['bigtop-apt-update'] -> Apt::Source<||> -> Package<||> - } else { - Apt::Conf<||> -> Apt::Key<||> -> Apt::Source<||> -> Package<||> - } + Exec['apt_update'] -> Package<||> + } default: { notify { "WARNING: running on a neither yum nor apt platform -- make sure Bigtop repo is setup": } diff --git a/bigtop-deploy/puppet/manifests/jdk.pp b/bigtop-deploy/puppet/manifests/jdk.pp index 895a81f502..36800eb0c0 100644 --- a/bigtop-deploy/puppet/manifests/jdk.pp +++ b/bigtop-deploy/puppet/manifests/jdk.pp @@ -41,7 +41,7 @@ noop => $jdk_preinstalled, } } - /(CentOS|Amazon|Fedora|RedHat)/: { + /(CentOS|Amazon|Fedora|RedHat|Rocky)/: { package { 'jdk': name => 'java-1.8.0-openjdk-devel', ensure => present, diff --git a/bigtop_toolchain/bin/puppetize.sh b/bigtop_toolchain/bin/puppetize.sh index 8bd3652325..6e9277dc98 100755 --- a/bigtop_toolchain/bin/puppetize.sh +++ b/bigtop_toolchain/bin/puppetize.sh @@ -20,51 +20,74 @@ if [ -f /etc/os-release ]; then . /etc/os-release fi +RPM_DOWNLOAD_URL="https://yum.puppet.com/puppet7" +APT_DOWNLOAD_URL="https://apt.puppet.com/puppet7" +NIGHTLY="" + +if [ "${ARCH}" = "ppc64le" ];then +RPM_DOWNLOAD_URL="https://nightlies.puppetlabs.com/yum/puppet7-nightly" +APT_DOWNLOAD_URL="https://nightlies.puppetlabs.com/apt/puppet7-nightly" +NIGHTLY="-nightly" +fi + case ${ID}-${VERSION_ID} in fedora-35) + rpm -Uvh ${RPM_DOWNLOAD_URL}-release-fedora-34.noarch.rpm dnf -y install yum-utils dnf -y check-update - dnf -y install hostname diffutils findutils curl sudo unzip wget puppet procps-ng libxcrypt-compat systemd + dnf -y install hostname diffutils findutils curl sudo unzip wget puppet-agent procps-ng libxcrypt-compat systemd # On Fedora 31, the puppetlabs-stdlib package provided by the distro installs the module # into /usr/share/puppet/modules, but it's not recognized as the default module path. # So we install that module in the same way as CentOS 7. - puppet module install puppetlabs-stdlib --version 4.12.0 + /opt/puppetlabs/bin/puppet module install puppetlabs-stdlib ;; ubuntu-18.04|ubuntu-20.04) apt-get update - apt-get -y install wget curl sudo unzip puppet software-properties-common puppet-module-puppetlabs-apt puppet-module-puppetlabs-stdlib systemd-sysv + apt-get -y install wget curl sudo unzip software-properties-common systemd-sysv + wget -P /tmp ${APT_DOWNLOAD_URL}-release-bionic.deb + dpkg -i /tmp/puppet7${NIGHTLY}-release-bionic.deb + apt-get update + apt-get -y install puppet-agent + /opt/puppetlabs/bin/puppet module install puppetlabs-stdlib + /opt/puppetlabs/bin/puppet module install puppetlabs-apt ;; debian-10*|debian-11*) apt-get update - apt-get -y install wget curl sudo unzip puppet puppet-module-puppetlabs-apt puppet-module-puppetlabs-stdlib systemd-sysv gnupg procps + apt-get -y install wget curl sudo unzip systemd-sysv gnupg procps + wget -P /tmp ${APT_DOWNLOAD_URL}-release-buster.deb + dpkg -i /tmp/puppet7${NIGHTLY}-release-buster.deb + apt-get update + apt-get -y install puppet-agent + /opt/puppetlabs/bin/puppet module install puppetlabs-stdlib + /opt/puppetlabs/bin/puppet module install puppetlabs-apt ;; centos-7*) rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - yum updateinfo - # BIGTOP-3088: pin puppetlabs-stdlib to 4.12.0 as the one provided by - # distro (4.25.0) has conflict with puppet<4. Should be removed once - # puppet in distro is updated. - yum -y install hostname curl sudo unzip wget puppet - puppet module install puppetlabs-stdlib --version 4.12.0 + rpm -Uvh ${RPM_DOWNLOAD_URL}-release-el-7.noarch.rpm + yum update -y + yum -y install hostname curl sudo unzip wget puppet-agent + # bump puppet to puppet7 + /opt/puppetlabs/bin/puppet module install puppetlabs-stdlib ;; centos-8*) sed -i -e 's/^\(mirrorlist\)/#\1/' -e 's,^#baseurl=http://mirror.centos.org,baseurl=https://vault.centos.org,' /etc/yum.repos.d/CentOS-Linux-* ;& rocky-8*) rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm + rpm -Uvh ${RPM_DOWNLOAD_URL}-release-el-8.noarch.rpm dnf -y check-update - dnf -y install glibc-langpack-en hostname diffutils curl sudo unzip wget puppet 'dnf-command(config-manager)' + dnf -y install glibc-langpack-en hostname diffutils curl sudo unzip wget puppet-agent 'dnf-command(config-manager)' # Install the module in the same way as Fedora 31 and CentOS 7 for compatibility issues. - puppet module install puppetlabs-stdlib --version 4.12.0 + /opt/puppetlabs/bin/puppet module install puppetlabs-stdlib # Enabling the PowerTools and EPEL repositories via Puppet doesn't seem to work in some cases. # As a workaround for that, enable the former here in advance of running the Puppet manifests. dnf config-manager --set-enabled powertools ;; rhel-8*) - rpm -Uvh https://yum.puppet.com/puppet5-release-el-8.noarch.rpm + rpm -Uvh ${RPM_DOWNLOAD_URL}-release-el-8.noarch.rpm dnf -y check-update dnf -y install hostname diffutils curl sudo unzip wget puppet-agent 'dnf-command(config-manager)' - puppet module install puppetlabs-stdlib + /opt/puppetlabs/bin/puppet module install puppetlabs-stdlib # Enabling the CodeReady repositories via Puppet doesn't seem to work in some cases. # As a workaround for that, enable the former here in advance of running the Puppet manifests. dnf config-manager --set-enabled codeready-builder-for-rhel-8-rhui-rpms diff --git a/bigtop_toolchain/manifests/puppet_modules.pp b/bigtop_toolchain/manifests/puppet_modules.pp index fc2f0e2aeb..b6d1c76b89 100644 --- a/bigtop_toolchain/manifests/puppet_modules.pp +++ b/bigtop_toolchain/manifests/puppet_modules.pp @@ -16,22 +16,15 @@ class bigtop_toolchain::puppet_modules { exec { 'install-puppet-stdlib': - path => '/usr/bin:/bin', - command => 'puppet module install puppetlabs-stdlib --version 4.12.0', - creates => '/etc/puppet/modules/stdlib', + command => '/opt/puppetlabs/bin/puppet module install puppetlabs-stdlib', + creates => '/etc/puppetlabs/code/environments/production/modules/stdlib', } case $operatingsystem{ /Ubuntu|Debian/: { - if versioncmp($::puppetversion, '4') < 0 { - $version = '--version 2.4.0' - } else { - $version = '' - } exec { 'install-puppet-apt': - path => '/usr/bin:/bin', - command => "puppet module install puppetlabs-apt ${version}", - creates => '/etc/puppet/modules/apt', + command => "/opt/puppetlabs/bin/puppet module install puppetlabs-apt", + creates => '/etc/puppetlabs/code/environments/production/modules/apt', } } } diff --git a/docker/bigtop-puppet/build.sh b/docker/bigtop-puppet/build.sh index 4cf4b0bd21..7944138be6 100755 --- a/docker/bigtop-puppet/build.sh +++ b/docker/bigtop-puppet/build.sh @@ -37,9 +37,9 @@ else ARCH="" fi -ENV_PATH="" -if [ ${OS} = "centos" -a ${VERSION} -ge 8 ]; then - ENV_PATH='ENV PATH /opt/puppetlabs/bin:$PATH' +ENV_PATH="ENV PATH /opt/puppetlabs/bin:$PATH" +if [ ${OS} = "debian" -a ${VERSION} -ge 10 ] || [ ${OS} = "ubuntu" ]; then + ENV_PATH='' fi cp ../../bigtop_toolchain/bin/puppetize.sh . diff --git a/docker/bigtop-slaves/Dockerfile.template b/docker/bigtop-slaves/Dockerfile.template index 6ff03ce0c1..b86122f8ea 100644 --- a/docker/bigtop-slaves/Dockerfile.template +++ b/docker/bigtop-slaves/Dockerfile.template @@ -19,7 +19,7 @@ MAINTAINER dev@bigtop.apache.org COPY bigtop_toolchain PUPPET_MODULES -RUN if [ -f ~/.bash_profile ]; then . ~/.bash_profile; fi && \ +RUN if [ -f /etc/profile.d/puppet-agent.sh ]; then . /etc/profile.d/puppet-agent.sh; fi && \ UPDATE_SOURCE && \ puppet apply -e "include bigtop_toolchain::installer" || if [ $? -ne 2 ]; then exit 1; fi COPY . /tmp/bigtop diff --git a/docker/bigtop-slaves/build.sh b/docker/bigtop-slaves/build.sh index 6514e1cb3a..a8d7ea2eb8 100755 --- a/docker/bigtop-slaves/build.sh +++ b/docker/bigtop-slaves/build.sh @@ -42,40 +42,22 @@ fi VERSION_INT=$(echo "$VERSION" | cut -d '.' -f 1) # setup puppet/modules path and update cmds +PUPPET_MODULES="/etc/puppetlabs/code/environments/production/modules/bigtop_toolchain" + case ${OS} in - ubuntu) - if [ "${VERSION_INT}" -gt "16" ]; then - PUPPET_MODULES="/usr/share/puppet/modules/bigtop_toolchain" - else - PUPPET_MODULES="/etc/puppet/modules/bigtop_toolchain" - fi - UPDATE_SOURCE="apt-get clean \&\& apt-get update" - ;; - debian) - PUPPET_MODULES="/usr/share/puppet/modules/bigtop_toolchain" + ubuntu|debian) UPDATE_SOURCE="apt-get clean \&\& apt-get update" ;; - fedora) - PUPPET_MODULES="/etc/puppet/code/modules/bigtop_toolchain" + fedora|rockylinux) UPDATE_SOURCE="dnf clean all \&\& dnf updateinfo" ;; centos) if [ "${VERSION_INT}" -gt "7" ]; then - PUPPET_MODULES="/etc/puppetlabs/code/environments/production/modules/bigtop_toolchain" UPDATE_SOURCE="dnf clean all \&\& dnf updateinfo" else - PUPPET_MODULES="/etc/puppet/modules/bigtop_toolchain" UPDATE_SOURCE="yum clean all \&\& yum updateinfo" fi ;; - rockylinux) - PUPPET_MODULES="/etc/puppetlabs/code/environments/production/modules/bigtop_toolchain" - UPDATE_SOURCE="dnf clean all \&\& dnf updateinfo" - ;; - opensuse) - PUPPET_MODULES="/etc/puppet/modules/bigtop_toolchain" - UPDATE_SOURCE="zypper clean \&\& zypper refresh" - ;; *) echo "[ERROR] Specified distro [${OS}] is not supported!" exit 1 diff --git a/provisioner/docker/docker-compose.yml b/provisioner/docker/docker-compose.yml index ffab94e5e5..866b515e4b 100644 --- a/provisioner/docker/docker-compose.yml +++ b/provisioner/docker/docker-compose.yml @@ -22,7 +22,7 @@ services: mem_limit: ${MEM_LIMIT} volumes: - ../../:/bigtop-home - - ./config/hiera.yaml:/etc/puppet/hiera.yaml - - ./config/hieradata:/etc/puppet/hieradata + - ./config/hiera.yaml:/etc/puppetlabs/code/environments/production/hiera.yaml + - ./config/hieradata:/etc/puppetlabs/code/environments/production/hieradata - ./config/hosts:/etc/hosts - /sys/fs/cgroup:/sys/fs/cgroup:ro diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh index 69b33c08d9..7b593ce6de 100755 --- a/provisioner/docker/docker-hadoop.sh +++ b/provisioner/docker/docker-hadoop.sh @@ -230,11 +230,11 @@ destroy() { } bigtop-puppet() { - if docker exec $1 bash -c "puppet --version" | grep ^3 >/dev/null ; then + if docker exec $1 bash -c "/opt/puppetlabs/bin/puppet --version" | grep ^3 >/dev/null ; then future="--parser future" fi - # BIGTOP-3401 Modify Puppet modulepath for puppetlabs-4.12 - docker exec $1 bash -c "puppet apply --detailed-exitcodes $future --hiera_config=/etc/puppet/hiera.yaml --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppet/modules:/usr/share/puppet/modules:/etc/puppetlabs/code/modules:/etc/puppet/code/modules /bigtop-home/bigtop-deploy/puppet/manifests" + # BIGTOP-3691. Bump Puppet to Puppet7 + docker exec $1 bash -c "/opt/puppetlabs/bin/puppet apply --detailed-exitcodes $future --hiera_config=/etc/puppetlabs/code/environments/production/hiera.yaml --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppetlabs/code/environments/production/modules:/etc/puppetlabs/code/modules /bigtop-home/bigtop-deploy/puppet/manifests" } get-yaml-config() {