From cc8c3e6faff7601fb7151c283586f68f2b18cc1e Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Wed, 19 Oct 2022 18:52:54 +0200 Subject: [PATCH 1/9] Fix typo in a spec test filename --- .../spec/defines/{users_acount_spec.rb => users_account_spec.rb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename puppet/spec/defines/{users_acount_spec.rb => users_account_spec.rb} (100%) diff --git a/puppet/spec/defines/users_acount_spec.rb b/puppet/spec/defines/users_account_spec.rb similarity index 100% rename from puppet/spec/defines/users_acount_spec.rb rename to puppet/spec/defines/users_account_spec.rb From a0c66780cfeed7df3e0ca006e39bcc6239a0edca Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Wed, 19 Oct 2022 14:30:19 +0200 Subject: [PATCH 2/9] Update saz/sudo to 7.x for EL8 support It is a major version due to dropping Puppet 5 but we're on 6 anyway. --- puppet/Puppetfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/Puppetfile b/puppet/Puppetfile index 2ca58ff6c..7f183bad3 100644 --- a/puppet/Puppetfile +++ b/puppet/Puppetfile @@ -37,7 +37,7 @@ mod 'puppetlabs/translate', '2.2.0' mod 'puppetlabs/vcsrepo', '3.1.0' mod 'puppetlabs/xinetd', '3.3.0' mod 'richardc/datacat', '0.6.2' -mod 'saz/sudo', '6.0.0' +mod 'saz/sudo', '7.0.2' mod 'theforeman/foreman', '13.1.0' mod 'theforeman/foreman_proxy', '12.1.0' mod 'theforeman/git', '6.0.1' From 1b68136d0f1be2ae7fc1de37193f23d5e9d0cc3d Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Fri, 16 Sep 2022 14:26:23 +0200 Subject: [PATCH 3/9] Update theforeman/* modules for Foreman 3.1 --- puppet/Puppetfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/puppet/Puppetfile b/puppet/Puppetfile index 7f183bad3..dec7e6164 100644 --- a/puppet/Puppetfile +++ b/puppet/Puppetfile @@ -15,11 +15,12 @@ mod 'puppet/jenkins', '3.0.0' mod 'puppet/letsencrypt', '8.0.2' mod 'puppet/nodejs', '8.0.0' mod 'puppet/pbuilder', '1.0.0' +mod 'puppet/redis', '8.4.0' mod 'puppet/rvm', '2.0.0' mod 'puppet/selinux', '3.1.0' mod 'puppet/systemd', '3.10.0' mod 'puppet/unattended_upgrades', '6.0.0' -mod 'puppetlabs/apache', '5.4.0' +mod 'puppetlabs/apache', '7.0.0' mod 'puppetlabs/apt', '7.7.0' mod 'puppetlabs/concat', '6.2.0 ' mod 'puppetlabs/inifile', '4.1.0 ' @@ -38,10 +39,11 @@ mod 'puppetlabs/vcsrepo', '3.1.0' mod 'puppetlabs/xinetd', '3.3.0' mod 'richardc/datacat', '0.6.2' mod 'saz/sudo', '7.0.2' -mod 'theforeman/foreman', '13.1.0' -mod 'theforeman/foreman_proxy', '12.1.0' +mod 'theforeman/foreman', '19.3.0' +mod 'theforeman/foreman_proxy', '21.0.0' mod 'theforeman/git', '6.0.1' mod 'theforeman/motd', '0.1.0' -mod 'theforeman/puppet', '12.0.1' +mod 'theforeman/puppet', '16.3.0' +mod 'theforeman/puppetserver_foreman', '2.0.0' mod 'theforeman/tftp', '5.0.1' mod 'treydock/yum_cron', '6.2.0' From 1d09d67cb8f480802479fcee271cc51a8de4caf5 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Fri, 16 Sep 2022 14:34:46 +0200 Subject: [PATCH 4/9] Update to Puppet 7 --- Vagrantfile | 10 +++++----- docs/jenkins.md | 4 ++-- docs/virt.md | 2 +- puppet/Gemfile | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index d892347d0..6df0d0377 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -8,7 +8,7 @@ Vagrant.configure("2") do |config| config.vm.provision "install puppet", type: "shell", inline: <<-SHELL . /etc/os-release yum -y install epel-release - yum -y install puppet6-release || yum -y install https://yum.puppetlabs.com/puppet6/puppet6-release-el-${VERSION_ID}.noarch.rpm + yum -y install puppet7-release || yum -y install https://yum.puppetlabs.com/puppet7/puppet7-release-el-${VERSION_ID}.noarch.rpm yum -y install puppet-agent SHELL @@ -59,8 +59,8 @@ Vagrant.configure("2") do |config| end override.vm.provision "install puppet", type: "shell", inline: <<-SHELL . /etc/os-release - wget https://apt.puppet.com/puppet6-release-${VERSION_CODENAME}.deb - apt-get install -y ./puppet6-release-${VERSION_CODENAME}.deb + wget https://apt.puppet.com/puppet7-release-${VERSION_CODENAME}.deb + apt-get install -y ./puppet7-release-${VERSION_CODENAME}.deb apt-get update apt-get install -y puppet-agent SHELL @@ -75,8 +75,8 @@ Vagrant.configure("2") do |config| end override.vm.provision "install puppet", type: "shell", inline: <<-SHELL . /etc/os-release - wget https://apt.puppet.com/puppet6-release-${VERSION_CODENAME}.deb - apt-get install -y ./puppet6-release-${VERSION_CODENAME}.deb + wget https://apt.puppet.com/puppet7-release-${VERSION_CODENAME}.deb + apt-get install -y ./puppet7-release-${VERSION_CODENAME}.deb apt-get update apt-get install -y puppet-agent SHELL diff --git a/docs/jenkins.md b/docs/jenkins.md index 4ac1a81d6..68498fae2 100644 --- a/docs/jenkins.md +++ b/docs/jenkins.md @@ -53,7 +53,7 @@ https://github.com/theforeman/foreman-infra/tree/master/puppet/modules contains For Enterprise Linux: * Ensure EPEL is configured: [epel-release](https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm) -* Ensure yum.puppet.com is configured: [puppet6-release](https://yum.puppet.com/puppet6/puppet6-release-el-7.noarch.rpm) +* Ensure yum.puppet.com is configured: [puppet7-release](https://yum.puppet.com/puppet7-release-el-7.noarch.rpm) * `yum -y install puppet-agent` * `echo "server = puppetmaster.theforeman.org" >> /etc/puppetlabs/puppet/puppet.conf` * ensure hostname is set node0X.jenkins..theforeman.org where is osuosl or aws for example and that the record is in DNS @@ -67,7 +67,7 @@ For Enterprise Linux: For Debian: -* Ensure apt.puppet.com is configured: [puppet6-release](https://apt.puppetlabs.com/puppet6-release-buster.deb) +* Ensure apt.puppet.com is configured: [puppet7-release](https://apt.puppetlabs.com/puppet7-release-bullseye.deb) * `apt update && apt install puppet-agent` * `echo "server = puppetmaster.theforeman.org" >> /etc/puppetlabs/puppet/puppet.conf` * Make the `puppet` command available: `source /etc/profile.d/puppet-agent.sh` diff --git a/docs/virt.md b/docs/virt.md index d537b847d..9c96953bc 100644 --- a/docs/virt.md +++ b/docs/virt.md @@ -37,7 +37,7 @@ systemctl enable --now libvirtd Now bootstrap Puppet: ``` -dnf install https://yum.puppet.com/puppet6-release-el-8.noarch.rpm +dnf install https://yum.puppet.com/puppet7-release-el-8.noarch.rpm dnf install puppet-agent . /etc/profile.d/puppet-agent.sh puppet config set server puppetmaster.theforeman.org diff --git a/puppet/Gemfile b/puppet/Gemfile index 522ab0338..8d7d91068 100644 --- a/puppet/Gemfile +++ b/puppet/Gemfile @@ -1,6 +1,6 @@ source 'https://rubygems.org' -gem 'puppet', '~> 6.9', require: false +gem 'puppet', '~> 7.0', require: false gem 'puppet-lint', require: false gem 'puppet-lint-absolute_classname-check', require: false gem 'puppet-lint-classes_and_types_beginning_with_digits-check', require: false From 606da54465f5281cb7bc7450fc65142e306b6d85 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Fri, 16 Sep 2022 14:35:15 +0200 Subject: [PATCH 5/9] Add a script to verify the dependencies --- .github/workflows/main.yml | 2 + puppet/Gemfile | 3 + puppet/check_dependencies | 134 +++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100755 puppet/check_dependencies diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 51b9adeb4..f570da62e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,6 +27,8 @@ jobs: unzip g10k-linux-amd64.zip - name: Install modules using g10k run: ./g10k -cachedir .g10k/cache -puppetfile + - name: Verify dependencies are compatible + run: bundle exec ./check_dependencies - name: Run syntax run: bundle exec rake syntax - name: Run lint diff --git a/puppet/Gemfile b/puppet/Gemfile index 8d7d91068..2d277646a 100644 --- a/puppet/Gemfile +++ b/puppet/Gemfile @@ -1,5 +1,8 @@ source 'https://rubygems.org' +gem 'puppet_metadata' +gem 'semantic_puppet' + gem 'puppet', '~> 7.0', require: false gem 'puppet-lint', require: false gem 'puppet-lint-absolute_classname-check', require: false diff --git a/puppet/check_dependencies b/puppet/check_dependencies new file mode 100755 index 000000000..8f346449b --- /dev/null +++ b/puppet/check_dependencies @@ -0,0 +1,134 @@ +#!/usr/bin/env ruby + +require 'puppet_metadata' +require 'semantic_puppet' + +def normalize_name(name) + name.tr('-', '/') +end + +class LocalModules < SemanticPuppet::Dependency::Source + attr_reader :modules + + def initialize(module_dirs) + @modules = {} + + module_dirs.each do |module_dir| + Dir[File.join(module_dir, '*', 'metadata.json')].sort.map do |metadata| + mod = PuppetMetadata.read(metadata) + dependencies = mod.dependencies.map { |name, requirement| [normalize_name(name), requirement.to_s] } + name = normalize_name(mod.name) + @modules[name] = create_release(name, mod.version, dependencies) + rescue PuppetMetadata::InvalidMetadataException => e + raise "#{metadata}: #{e}" + end + end + + @modules.freeze + end + + def fetch(name) + if @modules.key?(name) + [@modules[name]] + else + [] + end + end +end + +class ProvidedModules < SemanticPuppet::Dependency::Source + attr_reader :modules + + def initialize(modules) + @modules = {} + + modules.each do |name, version| + name = normalize_name(name) + @modules[name] = create_release(name, version) + end + + @modules.freeze + end + + def fetch(name) + if @modules.key?(name) + [@modules[name]] + else + [] + end + end +end + +# TODO: read this from environment.conf +local_source = LocalModules.new(['modules', 'external_modules']) +SemanticPuppet::Dependency.add_source(local_source) + +# TODO: Read this from a file +# These should be soft dependencies +provided = { + 'puppet/zypprepo' => '3.1.0', # puppet/jenkins + 'theforeman/dhcp' => '8.0.0', # theforeman/foreman_proxy + 'theforeman/dns' => '9.0.0', # theforeman/foreman_proxy +} +provided_source = ProvidedModules.new(provided) +SemanticPuppet::Dependency.add_source(provided_source) + +modules = Hash[local_source.modules.map { |name, mod| [name, mod.version.to_s] }] + +graph = SemanticPuppet::Dependency.query(modules) +begin + releases = SemanticPuppet::Dependency.resolve(graph) + puts "Satisfied all dependencies:" + releases.each do |release| + # TODO: print source + puts " #{release.name} #{release.version}" + end +rescue SemanticPuppet::Dependency::UnsatisfiableGraph => e + # There was some dependency that wasn't matched. We know its name and that + # it's a dependency we requested previously. Now to investigate why to + # provide useful hints + + # TODO: it's *very* weird to store unsatisfiable in a module + # Strange API + unsatisfiable = SemanticPuppet::Dependency.unsatisfiable + + unless unsatisfiable + # In older semantic_puppet unsatisfiable was unreliable + # Needs https://github.com/puppetlabs/semantic_puppet/pull/37 + $stderr.puts e + $stderr.puts "semantic_puppet provides insufficient information to hint why" + exit 1 + end + + $stderr.puts "Unable to satisfy #{unsatisfiable}" + + releases = graph.dependencies[unsatisfiable] + if releases.empty? + # This should never happen + $stderr.puts "No releases found for #{unsatisfiable}" + exit 2 + end + + releases.each do |release| + $stderr.puts "Investigating version #{release.version}" + unsatisfied = release.dependencies.select { |_, candidates| candidates.empty? } + if unsatisfied.empty? + # Is this ever the case, did we miss anything else? + $stderr.puts " All dependencies satisfied" + else + unsatisfied.each do |dependency, _| + constraint = release.constraints_for(dependency).find { |constraint| constraint[:source] == 'initialize' } + $stderr.puts " Dependency #{dependency} (#{constraint[:description]}) can't be satified" + + constraint_graph = SemanticPuppet::Dependency.query(dependency => '>= 0') + begin + releases = SemanticPuppet::Dependency.resolve(constraint_graph) + mod_releases = releases.select { |release| release.name == dependency } + $stderr.puts " Available versions: #{mod_releases.map { |r| r.version }.join(', ')}" + rescue SemanticPuppet::Dependency::UnsatisfiableGraph + $stderr.puts " Unable to find any release" + end + end + end + end +end From 6cbe2656dd605d22475cc4dcdf5fc3f84d0d3072 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Fri, 16 Sep 2022 14:36:28 +0200 Subject: [PATCH 6/9] Add a Hiera structure This allows deployment without the Foreman ENC. It does not manage users nor include utility. Those accept parameters with data that we do not want in Hiera due to privacy (user email addresses) and password hashes. Those should still be set via Foreman's ENC. --- puppet/data/common.yaml | 26 ++++++++++++ puppet/data/osfamily/RedHat-8.yaml | 9 +++++ puppet/hiera.yaml | 29 ++++++++++++++ puppet/manifests/site.pp | 40 +++++++++++++++++++ puppet/modules/deploy/manifests/init.pp | 12 +++++- puppet/modules/profiles/manifests/base.pp | 4 +- puppet/modules/profiles/manifests/foreman.pp | 14 +++++++ .../profiles/manifests/puppetserver.pp | 17 ++++++++ puppet/modules/utility/manifests/init.pp | 3 -- puppet/spec/classes/profiles_foreman_spec.rb | 11 +++++ .../classes/profiles_puppetserver_spec.rb | 18 +++++++++ 11 files changed, 176 insertions(+), 7 deletions(-) create mode 100644 puppet/data/common.yaml create mode 100644 puppet/data/osfamily/RedHat-8.yaml create mode 100644 puppet/hiera.yaml create mode 100644 puppet/manifests/site.pp create mode 100644 puppet/modules/profiles/manifests/foreman.pp create mode 100644 puppet/modules/profiles/manifests/puppetserver.pp create mode 100644 puppet/spec/classes/profiles_foreman_spec.rb create mode 100644 puppet/spec/classes/profiles_puppetserver_spec.rb diff --git a/puppet/data/common.yaml b/puppet/data/common.yaml new file mode 100644 index 000000000..51ca3caeb --- /dev/null +++ b/puppet/data/common.yaml @@ -0,0 +1,26 @@ +foreman_servicename: 'foreman.theforeman.org' +foreman_url: 'https://%{lookup("foreman_servicename")}' +puppet_servicename: 'puppet.theforeman.org' + +foreman::repo::repo: '3.1' +# TODO: theforeman/foreman 21.1.0 includes /server-status +foreman::config::apache::proxy_no_proxy_uris: + - /icons + - /server-status +foreman::serveraliases: + - '%{alias("foreman_servicename")}' + +foreman_proxy::trusted_hosts: + - "foreman01.conova.theforeman.org" +foreman_proxy::foreman_base_url: '%{alias("foreman_url")}' + +puppet::runmode: cron +puppet::puppetmaster: '%{alias("puppet_servicename")}' +puppet::server_additional_settings: + dns_alt_names: + - '%{alias("puppet_servicename")}' +puppet::server_foreman_url: '%{alias("foreman_url")}' +puppet::server_environments_owner: 'deploypuppet' +puppet::server_environments_group: 'deploypuppet' +puppet::server_puppetserver_telemetry: false +puppet::show_diff: true diff --git a/puppet/data/osfamily/RedHat-8.yaml b/puppet/data/osfamily/RedHat-8.yaml new file mode 100644 index 000000000..c44fee5f8 --- /dev/null +++ b/puppet/data/osfamily/RedHat-8.yaml @@ -0,0 +1,9 @@ +apache::default_mods: + # https://github.com/puppetlabs/puppetlabs-apache/pull/2337 + - http2 + - status +apache::protocols: + - 'h2' + - 'h2c' + - 'http/1.1' +apache::mpm_module: event diff --git a/puppet/hiera.yaml b/puppet/hiera.yaml new file mode 100644 index 000000000..f45ad1473 --- /dev/null +++ b/puppet/hiera.yaml @@ -0,0 +1,29 @@ +--- +version: 5 +defaults: # Used for any hierarchy level that omits these keys. + # The default value for "datadir" is "data" under the same directory as the hiera.yaml + # file (this file) + # When specifying a datadir, make sure the directory exists. + # See https://puppet.com/docs/puppet/latest/environments_about.html for further details on environments. + datadir: data + data_hash: yaml_data + +hierarchy: + - name: "Per-node data" + path: "nodes/%{trusted.certname}.yaml" + + - name: "Per-OS major defaults" + path: "os/%{facts.os.name}-%{facts.os.release.major}.yaml" + + - name: "Per-OS defaults" + path: "os/%{facts.os.name}.yaml" + + - name: "Per-OS family major defaults" + path: "osfamily/%{facts.os.family}-%{facts.os.release.major}.yaml" + + - name: "Per-OS family defaults" + path: "osfamily/%{facts.os.family}.yaml" + + - name: "Other YAML hierarchy levels" + paths: + - "common.yaml" diff --git a/puppet/manifests/site.pp b/puppet/manifests/site.pp new file mode 100644 index 000000000..28d7f43c1 --- /dev/null +++ b/puppet/manifests/site.pp @@ -0,0 +1,40 @@ +node default { + include profiles::base +} + +node /^controller\d+\.[a-z]+\.theforeman\.org$/ { + include profiles::base + include profiles::jenkins::controller +} + +node /^discourse\d+\.[a-z]+\.theforeman\.org$/ { + include profiles::base + # TODO profiles::discourse +} + +node /^foreman\d+\.[a-z]+\.theforeman\.org$/ { + include profiles::base + include profiles::foreman +} + +node /^node\d+\.jenkins\.[a-z]+\.theforeman\.org$/ { + include profiles::base + include profiles::jenkins::node +} + +node /^puppet\d+\.[a-z]+\.theforeman\.org$/ { + include profiles::base + include profiles::puppetserver +} + +node /^redmine\d+\.[a-z]+\.theforeman\.org$/ { + include profiles::base + include dirvish::client + include exim # TODO + include redmine +} + +node /^web\d+\.[a-z]+\.theforeman\.org$/ { + include profiles::base + include profiles::web +} diff --git a/puppet/modules/deploy/manifests/init.pp b/puppet/modules/deploy/manifests/init.pp index 9e47c410d..66bd2759a 100644 --- a/puppet/modules/deploy/manifests/init.pp +++ b/puppet/modules/deploy/manifests/init.pp @@ -1,6 +1,14 @@ -class deploy { +# @summary Set up a deployment of Puppet environments +# +# @param user +# The username for the user +class deploy ( + String[1] $user = 'deploypuppet', +) { + # TODO: install g10k in $PATH + secure_ssh::receiver_setup { 'deploy': - user => 'deploypuppet', + user => $user, foreman_search => 'host ~ node*.jenkins.osuosl.theforeman.org and (name = external_ip4 or name = external_ip6)', script_content => file('deploy/script.sh'), } diff --git a/puppet/modules/profiles/manifests/base.pp b/puppet/modules/profiles/manifests/base.pp index 9b0a014fa..808292b10 100644 --- a/puppet/modules/profiles/manifests/base.pp +++ b/puppet/modules/profiles/manifests/base.pp @@ -1,11 +1,11 @@ # @summary A base profile for all machines class profiles::base ( ) { + include motd include puppet include ssh include timezone - include users - include utility + include unattended if $facts['os']['family'] == 'RedHat' and versioncmp($facts['os']['release']['major'], '8') >= 0 { include chrony } else { diff --git a/puppet/modules/profiles/manifests/foreman.pp b/puppet/modules/profiles/manifests/foreman.pp new file mode 100644 index 000000000..229c6153c --- /dev/null +++ b/puppet/modules/profiles/manifests/foreman.pp @@ -0,0 +1,14 @@ +# @summary A Foreman application server +class profiles::foreman { + include foreman + include foreman::repo + include foreman::compute::libvirt + include foreman::compute::openstack + include foreman::plugin::ansible + include foreman::plugin::puppet + include foreman::plugin::remote_execution + + puppet::config::main { 'dns_alt_names': + value => $foreman::serveraliases, + } +} diff --git a/puppet/modules/profiles/manifests/puppetserver.pp b/puppet/modules/profiles/manifests/puppetserver.pp new file mode 100644 index 000000000..d028f93df --- /dev/null +++ b/puppet/modules/profiles/manifests/puppetserver.pp @@ -0,0 +1,17 @@ +# @summary A Puppetserver with Foreman integration +class profiles::puppetserver { + include puppet + include puppet::server + + include foreman::repo + class { 'foreman_proxy': + puppet => true, + puppetca => true, + } + include foreman_proxy::plugin::ansible + include foreman_proxy::plugin::remote_execution::ssh + + class { 'deploy': + user => $puppet::server_environments_owner, + } +} diff --git a/puppet/modules/utility/manifests/init.pp b/puppet/modules/utility/manifests/init.pp index bb6b6549d..80fca3aae 100644 --- a/puppet/modules/utility/manifests/init.pp +++ b/puppet/modules/utility/manifests/init.pp @@ -1,8 +1,5 @@ # Various basic utilities class utility($sysadmins = ['/dev/null']) { - include motd - include unattended - $vim = $facts['os']['family'] ? { 'RedHat' => 'vim-enhanced', default => 'vim', diff --git a/puppet/spec/classes/profiles_foreman_spec.rb b/puppet/spec/classes/profiles_foreman_spec.rb new file mode 100644 index 000000000..69b3ad773 --- /dev/null +++ b/puppet/spec/classes/profiles_foreman_spec.rb @@ -0,0 +1,11 @@ +require 'spec_helper' + +describe 'profiles::foreman' do + on_supported_os.each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + it { is_expected.to compile.with_all_deps } + end + end +end diff --git a/puppet/spec/classes/profiles_puppetserver_spec.rb b/puppet/spec/classes/profiles_puppetserver_spec.rb new file mode 100644 index 000000000..d860848c0 --- /dev/null +++ b/puppet/spec/classes/profiles_puppetserver_spec.rb @@ -0,0 +1,18 @@ +require 'spec_helper' + +describe 'profiles::puppetserver' do + on_supported_os.each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + let(:pre_condition) do + <<~PUPPET + class { 'puppet': + server_environments_owner => 'deploypuppet', + } + PUPPET + end + + it { is_expected.to compile.with_all_deps } + end + end +end From 6dfb0dc90999dddefb95db6068c4c501df7fb485 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Tue, 20 Sep 2022 18:36:45 +0200 Subject: [PATCH 7/9] Add documentation for the new Foreman/Puppet setup --- docs/bootstrap.md | 61 ++++++++++++++++++++++++++++++++++++ docs/foreman-puppetserver.md | 20 ------------ docs/foreman.md | 10 ++++++ docs/index.md | 3 +- docs/puppet.md | 10 ++++++ mkdocs.yml | 4 ++- 6 files changed, 86 insertions(+), 22 deletions(-) create mode 100644 docs/bootstrap.md delete mode 100644 docs/foreman-puppetserver.md create mode 100644 docs/foreman.md create mode 100644 docs/puppet.md diff --git a/docs/bootstrap.md b/docs/bootstrap.md new file mode 100644 index 000000000..7b17d502c --- /dev/null +++ b/docs/bootstrap.md @@ -0,0 +1,61 @@ +# Boostrap a new environment + +To rebuild the whole Foreman Infrastructure from scratch, these are the steps to get started. + +## Build a Puppetserver + +Install a minimal EL8 install. Then: + +```sh +dnf install https://yum.puppet.com/puppet7-release-el-8.noarch.rpm +dnf install puppetserver +. /etc/profile.d/puppet-agent.sh +puppetserver ca setup --ca-name 'Foreman Puppet CA' --certname $HOSTNAME --subject-alt-names puppet.theforeman.org +puppet config set --section agent server puppet.theforeman.org +puppet config set --section main dns_alt_names puppet.theforeman.org +# To allow the foreman node a SAN +sed -i '/allow-subject-alt-names/ s/false/true/' /etc/puppetlabs/puppetserver/conf.d/ca.conf +systemctl enable --now puppetserver puppet +firewall-cmd --add-port 8140/tcp +firewall-cmd --add-port 8140/tcp --permanent +mkdir /etc/puppetlabs/code/environments/bootstrap +chown YOURUSER: /etc/puppetlabs/code/environments/bootstrap +``` + +With a basic Puppetserver running, deploy the environment. From your local machine: + +``` +# Download the latest from https://github.com/xorpaul/g10k/releases and place it in ~/bin +git clone https://github.com/theforeman/foreman-infra +cd foreman-infra/puppet +g10k -cachedir ~/.cache/.g10k -puppetfile +rsync -av --delete --exclude={Gem,Rake,Puppet}file*,test_modules,spec,check_dependencies ./ SERVER.EXAMPLE.COM:/etc/puppetlabs/code/environments/bootstrap/ +``` + +## Build a Foreman server + +Install a minimal EL8 install. Then: + +```sh +dnf install https://yum.puppet.com/puppet7-release-el-8.noarch.rpm +dnf install puppet-agent +. /etc/profile.d/puppet-agent.sh +puppet config set --section agent environment bootstrap +puppet config set --section agent server puppet.theforeman.org +puppet config set --section main dns_alt_names foreman.theforeman.org +puppet ssl bootstrap +systemctl enable --now puppet +``` + +In case it should become a production setup, import the database dump: +``` +puppet agent --disable +systemctl stop foreman\* dynflow\* +sudo -u postgres dropdb foreman +sudo -u postgres createdb -O foreman foreman +sudo -u postgres psql foreman < /path/to/dump.sql +foreman-rake db:migrate +foreman-rake db:seed +puppet agent --enable +puppet agent -t +``` diff --git a/docs/foreman-puppetserver.md b/docs/foreman-puppetserver.md deleted file mode 100644 index 02957a97a..000000000 --- a/docs/foreman-puppetserver.md +++ /dev/null @@ -1,20 +0,0 @@ -# Foreman and Puppet Server - -Foreman and Puppet are used to manage every machine in the Foreman infrastructure. The Foreman instance is accessible only to those with SSH access to puppetmaster.theforeman.org. Add the following snippet to `~/.ssh/config`: - -``` -Host foreman-pm - HostName puppetmaster.theforeman.org - Port 8122 - User - LocalForward 9443 localhost:443 - ExitOnForwardFailure yes -``` - -and then run: - -``` -ssh foreman-pm -``` - -and open https://localhost:9443 in your browser. diff --git a/docs/foreman.md b/docs/foreman.md new file mode 100644 index 000000000..de1d82ddd --- /dev/null +++ b/docs/foreman.md @@ -0,0 +1,10 @@ +# Foreman + +| | foreman01.conova.theforeman.org | +| - | - | +| type | Libvirt VM | +| OS | CentOS Stream 8 | +| CPUs | 4 | +| RAM | 4GB | +| Storage | /dev/vda (20GB) | +| Managed by | [profiles::foreman](https://github.com/theforeman/foreman-infra/blob/master/puppet/modules/profiles/manifests/foreman.pp) | diff --git a/docs/index.md b/docs/index.md index 37dfffc2d..99532a1b7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -18,10 +18,11 @@ Fork https://github.com/theforeman/foreman-infra and add your key into the [file | Role | Provider(s) | |---|---| | [Discourse](discourse.md) | Scaleway | -| [Foreman](foreman-puppetserver.md) | Scaleway | +| [Foreman](foreman.md) | Conova | | [Koji](koji.md) | AWS | | [Jenkins](jenkins.md) | OSUOSL | | [Jenkins Nodes](jenkins.md) | OSUOSL / AWS / Conova / Netways | +| [Puppet](puppet.md) | Conova | | [Redmine](redmine.md) | Scaleway | | [Virt](virt.md) | Conova | | [Webserver](webserver.md) | OSUOSL | diff --git a/docs/puppet.md b/docs/puppet.md new file mode 100644 index 000000000..16220e2ac --- /dev/null +++ b/docs/puppet.md @@ -0,0 +1,10 @@ +# Puppetserver + +| | puppet01.conova.theforeman.org | +| - | - | +| type | Libvirt VM | +| OS | CentOS Stream 8 | +| CPUs | 4 | +| RAM | 8GB | +| Storage | /dev/vda (20GB) | +| Managed by | [profiles::puppetserver](https://github.com/theforeman/foreman-infra/blob/master/puppet/modules/profiles/manifests/puppetserver.pp) | diff --git a/mkdocs.yml b/mkdocs.yml index d801e1692..ba5c3990f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,10 +4,12 @@ nav: - Overview: index.md - Roles: - Discourse: discourse.md - - Foreman: foreman-puppetserver.md + - Foreman: foreman.md - Jenkins: jenkins.md - Koji: koji.md + - Puppet: puppet.md - Redmine: redmine.md - Virt: virt.md - Webserver: webserver.md - GPG: gpg.md + - Bootstrap: bootstrap.md From f9243fcd65a874bcf56091eb8b00a70100831ec2 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Tue, 18 Oct 2022 15:18:55 +0200 Subject: [PATCH 8/9] Set the Foreman URL in foreman searches Also switches to the non-deprecated foreman::foreman method. --- puppet/modules/secure_ssh/manifests/receiver_setup.pp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/puppet/modules/secure_ssh/manifests/receiver_setup.pp b/puppet/modules/secure_ssh/manifests/receiver_setup.pp index 664f7b551..97dfac036 100644 --- a/puppet/modules/secure_ssh/manifests/receiver_setup.pp +++ b/puppet/modules/secure_ssh/manifests/receiver_setup.pp @@ -57,12 +57,7 @@ if $foreman_search and defined('$::foreman_api_user') and defined('$::foreman_api_password') { # Get the IPs of the uploaders from foreman - $ip_data=foreman({ - 'item' => 'fact_values', - 'search' => $foreman_search, - 'foreman_user' => $::foreman_api_user, - 'foreman_pass' => $::foreman_api_password, - }) + $ip_data = foreman::foreman('fact_values', $foreman_search, '20', lookup('foreman_url'), $::foreman_api_user, $::foreman_api_password) } file { "${homedir}/.ssh/authorized_keys": From 32fd91e5a647a4cf8b8c5c9d25814e3ffe38288f Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Wed, 19 Oct 2022 14:24:22 +0200 Subject: [PATCH 9/9] Use system groups for sudo access On Debian there's the sudo group which by default can use password access. For Red Hat the wheel group does the same. This saves individual sudo entries. The only exception is the jenkins user on slave which can use passwordless sudo. For that a specific entry is made. --- puppet/data/common.yaml | 2 ++ puppet/modules/slave/manifests/init.pp | 10 ++++++---- puppet/modules/users/manifests/account.pp | 23 ++++++++++------------- puppet/spec/classes/slave_spec.rb | 6 ++++-- puppet/spec/defines/users_account_spec.rb | 8 ++++---- 5 files changed, 26 insertions(+), 23 deletions(-) diff --git a/puppet/data/common.yaml b/puppet/data/common.yaml index 51ca3caeb..b6ebfa93a 100644 --- a/puppet/data/common.yaml +++ b/puppet/data/common.yaml @@ -24,3 +24,5 @@ puppet::server_environments_owner: 'deploypuppet' puppet::server_environments_group: 'deploypuppet' puppet::server_puppetserver_telemetry: false puppet::show_diff: true + +sudo::wheel_config: password diff --git a/puppet/modules/slave/manifests/init.pp b/puppet/modules/slave/manifests/init.pp index fc4c9bb4d..bc0e2ba34 100644 --- a/puppet/modules/slave/manifests/init.pp +++ b/puppet/modules/slave/manifests/init.pp @@ -26,14 +26,16 @@ include git # On Debian we use pbuilder with sudo - $sudo = $facts['os']['family'] ? { - 'Debian' => 'ALL=NOPASSWD: ALL', - default => '', + if $facts['os']['family'] == 'Debian' { + include sudo + sudo::conf { "sudo-puppet-${username}": + content => "${username} ALL=NOPASSWD: ALL", + } } users::account { $username: homedir => $homedir, - sudo => $sudo, + sudo => false, } file { $workspace: diff --git a/puppet/modules/users/manifests/account.pp b/puppet/modules/users/manifests/account.pp index 540e70ea7..0f9f92900 100644 --- a/puppet/modules/users/manifests/account.pp +++ b/puppet/modules/users/manifests/account.pp @@ -1,14 +1,22 @@ -define users::account( +define users::account ( Enum['present', 'absent'] $ensure = 'present', Optional[String] $fullname = undef, Optional[String] $passwd = undef, Stdlib::Absolutepath $homedir = "/home/${title}", - String $sudo = 'ALL=(ALL) ALL', + Boolean $sudo = true, ) { + if $sudo { + include sudo + $groups = [if $facts['os']['family'] == 'Debian' { 'sudo' } else { 'wheel'}] + } else { + $groups = [] + } + user { $name: ensure => $ensure, comment => $fullname, home => $homedir, + groups => $groups, managehome => true, shell => '/bin/bash', password => $passwd, @@ -36,16 +44,5 @@ group => $name, mode => '0600', } - - $sudo_ensure = bool2str($sudo == '', 'absent', 'present') - } else { - $sudo_ensure = $ensure - } - - include sudo - sudo::conf { "sudo-puppet-${name}": - ensure => $sudo_ensure, - content => "${name} ${sudo}", } - } diff --git a/puppet/spec/classes/slave_spec.rb b/puppet/spec/classes/slave_spec.rb index 35839fda7..03452b5e4 100644 --- a/puppet/spec/classes/slave_spec.rb +++ b/puppet/spec/classes/slave_spec.rb @@ -12,10 +12,12 @@ {uploader: false} end it { is_expected.to compile.with_all_deps } + it { is_expected.to contain_users__account('jenkins').with_sudo(false) } if facts[:osfamily] == 'Debian' - it { is_expected.to contain_users__account('jenkins').with_sudo('ALL=NOPASSWD: ALL') } + it { is_expected.to contain_class('sudo') } + it { is_expected.to contain_sudo__conf('sudo-puppet-jenkins').with_content('jenkins ALL=NOPASSWD: ALL') } else - it { is_expected.to contain_users__account('jenkins').with_sudo('') } + it { is_expected.not_to contain_class('sudo') } end if facts[:osfamily] == 'Debian' diff --git a/puppet/spec/defines/users_account_spec.rb b/puppet/spec/defines/users_account_spec.rb index 3d965a637..41c1a8b18 100644 --- a/puppet/spec/defines/users_account_spec.rb +++ b/puppet/spec/defines/users_account_spec.rb @@ -5,20 +5,21 @@ context "on #{os}" do let(:title) { 'jenkins' } let(:facts) { facts } + let(:sudo_group) { facts[:os]['family'] == 'Debian' ? 'sudo' : 'wheel' } context 'default parameters' do it { is_expected.to compile.with_all_deps } - it { is_expected.to contain_user('jenkins').with_ensure('present') } + it { is_expected.to contain_user('jenkins').with_ensure('present').with_groups([sudo_group]) } it { is_expected.to contain_file('/home/jenkins').with_ensure('directory') } end context 'without sudo' do let(:params) do - {sudo: ''} + {sudo: false} end it { is_expected.to compile.with_all_deps } - it { is_expected.to contain_sudo__conf('sudo-puppet-jenkins').with_ensure('absent') } + it { is_expected.to contain_user('jenkins').with_ensure('present').with_groups([]) } end context 'ensure => absent' do @@ -28,7 +29,6 @@ it { is_expected.to compile.with_all_deps } it { is_expected.to contain_user('jenkins').with_ensure('absent') } - it { is_expected.to contain_sudo__conf('sudo-puppet-jenkins').with_ensure('absent') } it { is_expected.not_to contain_file('/home/jenkins') } end end