diff --git a/REFERENCE.md b/REFERENCE.md index 01f75ca64..fca3e9b5c 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -30,6 +30,7 @@ * [`peadm::file_or_content`](#peadm--file_or_content) * [`peadm::flatten_compact`](#peadm--flatten_compact) * [`peadm::generate_pe_conf`](#peadm--generate_pe_conf): Generate a pe.conf file in JSON format +* [`peadm::get_node_group_environment`](#peadm--get_node_group_environment): check if a custom PE environment is set in pe.conf * [`peadm::get_pe_conf`](#peadm--get_pe_conf) * [`peadm::get_targets`](#peadm--get_targets): Accept undef or a SingleTargetSpec, and return an Array[Target, 1, 0]. This differs from get_target() in that: - It returns an Array[Target * [`peadm::log_plan_parameters`](#peadm--log_plan_parameters) @@ -740,6 +741,24 @@ Data type: `Hash` A hash of settings to set in the config file. Any keys that are set to undef will not be included in the config file. +### `peadm::get_node_group_environment` + +Type: Puppet Language + +check if a custom PE environment is set in pe.conf + +#### `peadm::get_node_group_environment(Peadm::SingleTargetSpec $primary)` + +The peadm::get_node_group_environment function. + +Returns: `String` the desired environment for PE specific node groups + +##### `primary` + +Data type: `Peadm::SingleTargetSpec` + +the FQDN for the primary, here we will read the pe.conf from + ### `peadm::get_pe_conf` Type: Puppet Language @@ -1832,12 +1851,21 @@ The peadm::add_database class. The following parameters are available in the `peadm::add_database` plan: +* [`node_group_environment`](#-peadm--add_database--node_group_environment) * [`targets`](#-peadm--add_database--targets) * [`primary_host`](#-peadm--add_database--primary_host) * [`mode`](#-peadm--add_database--mode) * [`begin_at_step`](#-peadm--add_database--begin_at_step) * [`is_migration`](#-peadm--add_database--is_migration) +##### `node_group_environment` + +Data type: `String[1]` + +environment for the PEADM specific node groups, if not set it will be gathered from pe.conf or production + +Default value: `peadm::get_node_group_environment($primary_host)` + ##### `targets` Data type: `Peadm::SingleTargetSpec` @@ -1899,6 +1927,7 @@ The following parameters are available in the `peadm::add_replica` plan: * [`replica_host`](#-peadm--add_replica--replica_host) * [`replica_postgresql_host`](#-peadm--add_replica--replica_postgresql_host) * [`token_file`](#-peadm--add_replica--token_file) +* [`node_group_environment`](#-peadm--add_replica--node_group_environment) ##### `primary_host` @@ -1929,6 +1958,14 @@ Data type: `Optional[String]` Default value: `undef` +##### `node_group_environment` + +Data type: `String[1]` + +environment for the PEADM specific node groups, if not set it will be gathered from pe.conf or production + +Default value: `peadm::get_node_group_environment($primary_host)` + ### `peadm::backup` Backup puppet primary configuration @@ -2016,6 +2053,7 @@ management using PEAdm. The following parameters are available in the `peadm::convert` plan: * [`begin_at_step`](#-peadm--convert--begin_at_step) +* [`node_group_environment`](#-peadm--convert--node_group_environment) * [`primary_host`](#-peadm--convert--primary_host) * [`replica_host`](#-peadm--convert--replica_host) * [`compiler_hosts`](#-peadm--convert--compiler_hosts) @@ -2035,6 +2073,14 @@ The step where the plan should start. If not set, it will start at the beginning Default value: `undef` +##### `node_group_environment` + +Data type: `String[1]` + +environment for the PEADM specific node groups, if not set it will be gathered from pe.conf or production + +Default value: `peadm::get_node_group_environment($primary_host)` + ##### `primary_host` Data type: `Peadm::SingleTargetSpec` @@ -2723,6 +2769,7 @@ The following parameters are available in the `peadm::upgrade` plan: * [`stagingdir`](#-peadm--upgrade--stagingdir) * [`uploaddir`](#-peadm--upgrade--uploaddir) * [`begin_at_step`](#-peadm--upgrade--begin_at_step) +* [`node_group_environment`](#-peadm--upgrade--node_group_environment) * [`primary_host`](#-peadm--upgrade--primary_host) * [`replica_host`](#-peadm--upgrade--replica_host) * [`compiler_hosts`](#-peadm--upgrade--compiler_hosts) @@ -2819,6 +2866,14 @@ The step where the plan should start. If not set, it will start at the beginning Default value: `undef` +##### `node_group_environment` + +Data type: `String[1]` + +environment for the PEADM specific node groups, if not set it will be gathered from pe.conf or production + +Default value: `peadm::get_node_group_environment($primary_host)` + ##### `primary_host` Data type: `Peadm::SingleTargetSpec` diff --git a/functions/get_node_group_environment.pp b/functions/get_node_group_environment.pp new file mode 100644 index 000000000..2daf3b2e2 --- /dev/null +++ b/functions/get_node_group_environment.pp @@ -0,0 +1,33 @@ +# +# @summary check if a custom PE environment is set in pe.conf +# +# @param primary the FQDN for the primary, here we will read the pe.conf from +# +# @return [String] the desired environment for PE specific node groups +# +# @see https://www.puppet.com/docs/pe/latest/upgrade_pe#update_environment +# +# @author Tim Meusel +# +function peadm::get_node_group_environment(Peadm::SingleTargetSpec $primary) { + $peconf = peadm::get_pe_conf(get_target($primary)) + # if both are set, they need to be set to the same value + # if they are not set, we assume that the user runs their infra in production + $pe_install = $peconf['pe_install::install::classification::pe_node_group_environment'] + $puppet_enterprise = $peconf['puppet_enterprise::master::recover_configuration::pe_environment'] + + # check if both are equal + # This also evaluates to true if both are undef + if $pe_install == $puppet_enterprise { + # check if the option isn't undef + # ToDo: A proper regex for allowed characters in an environment would be nice + # https://github.com/puppetlabs/puppet-docs/issues/1158 + if $pe_install =~ String[1] { + return $pe_install + } else { + return 'production' + } + } else { + fail("pe_install::install::classification::pe_node_group_environment and puppet_enterprise::master::recover_configuration::pe_environment need to be set to the same value, not '${pe_install}' and '${puppet_enterprise}'") + } +} diff --git a/manifests/setup/legacy_compiler_group.pp b/manifests/setup/legacy_compiler_group.pp index 4eff95b6a..d2d9e69d5 100644 --- a/manifests/setup/legacy_compiler_group.pp +++ b/manifests/setup/legacy_compiler_group.pp @@ -3,9 +3,11 @@ String[1] $primary_host, Optional[String] $internal_compiler_a_pool_address = undef, Optional[String] $internal_compiler_b_pool_address = undef, + String[1] $node_group_environment = 'production', ) { Node_group { purge_behavior => none, + environment => $node_group_environment, } node_group { 'PE Legacy Compiler': diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index 9fcc336ba..ef28226a2 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -23,6 +23,7 @@ # A load balancer address directing traffic to any of the "B" pool # compilers. This is used for DR configuration in large and extra large # architectures. +# @param node_group_environment the environment that will be assigned to all the PE Infra node groups # class peadm::setup::node_manager ( String[1] $primary_host, @@ -36,6 +37,7 @@ Optional[String[1]] $compiler_pool_address = undef, Optional[String[1]] $internal_compiler_a_pool_address = $server_a_host, Optional[String[1]] $internal_compiler_b_pool_address = $server_b_host, + String[1] $node_group_environment = 'production', ) { # "Not-configured" placeholder string. This will be used in places where we # cannot set an explicit null, and need to supply some kind of value. @@ -46,6 +48,7 @@ # else. Node_group { purge_behavior => none, + environment => $node_group_environment, } ################################################## diff --git a/plans/add_database.pp b/plans/add_database.pp index 8ee62749f..ba1c32c82 100644 --- a/plans/add_database.pp +++ b/plans/add_database.pp @@ -1,3 +1,6 @@ +# +# @param node_group_environment environment for the PEADM specific node groups, if not set it will be gathered from pe.conf or production +# plan peadm::add_database( Peadm::SingleTargetSpec $targets, Peadm::SingleTargetSpec $primary_host, @@ -10,6 +13,7 @@ 'cleanup-db', 'finalize']] $begin_at_step = undef, Optional[Boolean] $is_migration = false, + String[1] $node_group_environment = peadm::get_node_group_environment($primary_host), ) { $primary_target = peadm::get_targets($primary_host, 1) $postgresql_target = peadm::get_targets($targets, 1) @@ -135,15 +139,17 @@ $host = pick($a_host, $b_host) out::verbose("In transitive state, setting classification to ${host}") run_plan('peadm::util::update_classification', $primary_target, - postgresql_a_host => $host, - postgresql_b_host => $host, - peadm_config => $peadm_config + postgresql_a_host => $host, + postgresql_b_host => $host, + peadm_config => $peadm_config, + node_group_environment => $node_group_environment, ) } else { run_plan('peadm::util::update_classification', $primary_target, - postgresql_a_host => $avail_group_letter ? { 'A' => $postgresql_host, default => undef }, - postgresql_b_host => $avail_group_letter ? { 'B' => $postgresql_host, default => undef }, - peadm_config => $peadm_config + postgresql_a_host => $avail_group_letter ? { 'A' => $postgresql_host, default => undef }, + postgresql_b_host => $avail_group_letter ? { 'B' => $postgresql_host, default => undef }, + peadm_config => $peadm_config, + node_group_environment => $node_group_environment, ) } } diff --git a/plans/add_replica.pp b/plans/add_replica.pp index c21d536fc..cef188188 100644 --- a/plans/add_replica.pp +++ b/plans/add_replica.pp @@ -7,6 +7,7 @@ # @param replica_postgresql_host - The hostname and certname of the host with the replica PE-PosgreSQL database. # Can be a separate host in an XL architecture, or undef in Standard or Large. # @param token_file - (optional) the token file in a different location than the default. +# @param node_group_environment environment for the PEADM specific node groups, if not set it will be gathered from pe.conf or production plan peadm::add_replica( # Standard or Large Peadm::SingleTargetSpec $primary_host, @@ -17,6 +18,7 @@ # Common Configuration Optional[String] $token_file = undef, + String[1] $node_group_environment = peadm::get_node_group_environment($primary_host), ) { $primary_target = peadm::get_targets($primary_host, 1) $replica_target = peadm::get_targets($replica_host, 1) @@ -96,7 +98,8 @@ server_b_host => $replica_avail_group_letter ? { 'B' => $replica_target.peadm::certname(), default => undef }, internal_compiler_a_pool_address => $replica_avail_group_letter ? { 'A' => $replica_target.peadm::certname(), default => undef }, internal_compiler_b_pool_address => $replica_avail_group_letter ? { 'B' => $replica_target.peadm::certname(), default => undef }, - peadm_config => $peadm_config + peadm_config => $peadm_config, + node_group_environment => $node_group_environment, ) # Source list of files on Primary and synchronize to new Replica diff --git a/plans/convert.pp b/plans/convert.pp index a18928937..051f89492 100644 --- a/plans/convert.pp +++ b/plans/convert.pp @@ -5,6 +5,7 @@ # management using PEAdm. # # @param begin_at_step The step where the plan should start. If not set, it will start at the beginning +# @param node_group_environment environment for the PEADM specific node groups, if not set it will be gathered from pe.conf or production # plan peadm::convert ( # Standard @@ -26,6 +27,7 @@ Array[String] $dns_alt_names = [], Optional[Peadm::ConvertSteps] $begin_at_step = undef, + String[1] $node_group_environment = peadm::get_node_group_environment($primary_host), ) { peadm::assert_supported_bolt_version() @@ -316,6 +318,7 @@ compiler_pool_address => $compiler_pool_address, internal_compiler_a_pool_address => $internal_compiler_a_pool_address, internal_compiler_b_pool_address => $internal_compiler_b_pool_address, + node_group_environment => $node_group_environment, require => Class['peadm::setup::node_manager_yaml'], } diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index c612684b9..6adbbd292 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -3,6 +3,7 @@ Peadm::SingleTargetSpec $primary_host, TargetSpec $legacy_hosts, Optional[Boolean] $remove_pdb = true, + String[1] $node_group_environment = peadm::get_node_group_environment($primary_host), ) { $primary_target = peadm::get_targets($primary_host, 1) $convert_legacy_compiler_targets = peadm::get_targets($legacy_hosts) @@ -34,6 +35,7 @@ }, internal_compiler_a_pool_address => getvar('cluster.params.internal_compiler_a_pool_address'), internal_compiler_b_pool_address => getvar('cluster.params.internal_compiler_b_pool_address'), + node_group_environment => $node_group_environment, require => Class['peadm::setup::node_manager_yaml'], } } diff --git a/plans/upgrade.pp b/plans/upgrade.pp index 00236eb47..d20f1a1c5 100644 --- a/plans/upgrade.pp +++ b/plans/upgrade.pp @@ -32,6 +32,8 @@ # for offline usage. # @param begin_at_step The step where the plan should start. If not set, it will start at the beginning # +# @param node_group_environment environment for the PEADM specific node groups, if not set it will be gathered from pe.conf or production +# plan peadm::upgrade ( # Standard Peadm::SingleTargetSpec $primary_host, @@ -61,6 +63,7 @@ Boolean $permit_unsafe_versions = false, Optional[Peadm::UpgradeSteps] $begin_at_step = undef, + String[1] $node_group_environment = peadm::get_node_group_environment($primary_host), ) { # Log parameters for debugging peadm::log_plan_parameters({ @@ -69,6 +72,7 @@ 'compiler_hosts' => $compiler_hosts, 'primary_postgresql_host' => $primary_postgresql_host, 'replica_postgresql_host' => $replica_postgresql_host, + 'node_group_environment' => $node_group_environment, 'version' => $version, }) @@ -361,6 +365,7 @@ compiler_pool_address => $compiler_pool_address, internal_compiler_a_pool_address => $internal_compiler_a_pool_address, internal_compiler_b_pool_address => $internal_compiler_b_pool_address, + node_group_environment => $node_group_environment, require => Class['peadm::setup::node_manager_yaml'], } } diff --git a/plans/util/update_classification.pp b/plans/util/update_classification.pp index 247a96321..a5018e60c 100644 --- a/plans/util/update_classification.pp +++ b/plans/util/update_classification.pp @@ -2,9 +2,12 @@ # # @summary Configure classification # +# @param node_group_environment environment for the PEADM specific node groups, if not set it will be gathered from pe.conf or production +# plan peadm::util::update_classification ( # Standard Peadm::SingleTargetSpec $targets, + String[1] $node_group_environment, Optional[Hash] $peadm_config = undef, Optional[Peadm::SingleTargetSpec] $server_a_host = undef, Optional[Peadm::SingleTargetSpec] $server_b_host = undef, @@ -76,6 +79,7 @@ compiler_pool_address => $new['params']['compiler_pool_address'], internal_compiler_a_pool_address => $new['params']['internal_compiler_a_pool_address'], internal_compiler_b_pool_address => $new['params']['internal_compiler_b_pool_address'], + node_group_environment => $node_group_environment, require => Class['peadm::setup::node_manager_yaml'], } } diff --git a/spec/functions/get_node_group_environment_spec.rb b/spec/functions/get_node_group_environment_spec.rb new file mode 100644 index 000000000..bdc446c3b --- /dev/null +++ b/spec/functions/get_node_group_environment_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'peadm::get_node_group_environment' do + include BoltSpec::BoltContext + around(:each) do |example| + in_bolt_context do + example.run + end + end + let(:primary) do + ['test-vm.puppet.vm'] + end + + context 'returns production on empty pe.conf' do + it do + expect_task('peadm::read_file').with_params('path' => '/etc/puppetlabs/enterprise/conf.d/pe.conf').always_return({ 'content' => '{}' }) + is_expected.to run.with_params(primary).and_return('production') + end + end + + context 'returns production on non-empty pe.conf with unrelated settings' do + it do + expect_task('peadm::read_file').with_params('path' => '/etc/puppetlabs/enterprise/conf.d/pe.conf').always_return({ 'content' => '{"foo": true}' }) + is_expected.to run.with_params(primary).and_return('production') + end + end + context 'returns environment from pe.conf when set twice correctly' do + it do + pe = '{"pe_install::install::classification::pe_node_group_environment": "foo", "puppet_enterprise::master::recover_configuration::pe_environment": "foo"}' + expect_task('peadm::read_file').with_params('path' => '/etc/puppetlabs/enterprise/conf.d/pe.conf').always_return({ 'content' => pe }) + is_expected.to run.with_params(primary).and_return('foo') + end + end + context 'fails when both PE options are different' do + it do + pe = '{"pe_install::install::classification::pe_node_group_environment": "foo", "puppet_enterprise::master::recover_configuration::pe_environment": "bar"}' + expect_task('peadm::read_file').with_params('path' => '/etc/puppetlabs/enterprise/conf.d/pe.conf').always_return({ 'content' => pe }) + is_expected.to run.with_params(primary).and_raise_error(Puppet::PreformattedError, +%r{pe_install::install::classification::pe_node_group_environment and puppet_enterprise::master::recover_configuration::pe_environment}) + end + end +end diff --git a/spec/plans/convert_spec.rb b/spec/plans/convert_spec.rb index 99e453a4c..89c4aa09f 100644 --- a/spec/plans/convert_spec.rb +++ b/spec/plans/convert_spec.rb @@ -19,7 +19,8 @@ allow_apply expect_task('peadm::cert_data').return_for_targets('primary' => trustedjson).be_called_times(2) - expect_task('peadm::read_file').always_return({ 'content' => '2021.7.9' }) + expect_task('peadm::read_file').with_params('path' => '/opt/puppetlabs/server/pe_version').always_return({ 'content' => '2021.7.9' }) + expect_task('peadm::read_file').with_params('path' => '/etc/puppetlabs/enterprise/conf.d/pe.conf').always_return({ 'content' => '{}' }) expect_task('peadm::get_group_rules').return_for_targets('primary' => { '_output' => '{"rules": []}' }) expect_task('peadm::node_group_unpin').with_targets('primary').with_params({ 'node_certnames' => ['pe_compiler_legacy'], 'group_name' => 'PE Master' }) expect_task('peadm::check_legacy_compilers').with_targets('primary').with_params({ 'legacy_compilers' => 'pe_compiler_legacy' }).return_for_targets('primary' => { '_output' => '' }) diff --git a/spec/plans/upgrade_spec.rb b/spec/plans/upgrade_spec.rb index b92b828fa..5291906cd 100644 --- a/spec/plans/upgrade_spec.rb +++ b/spec/plans/upgrade_spec.rb @@ -37,6 +37,7 @@ def allow_standard_non_returning_calls expect_task('peadm::cert_data').return_for_targets('primary' => trusted_primary).be_called_times(1) expect_task('peadm::check_pe_master_rules').always_return(pe_rule_check) + expect_task('peadm::read_file').with_params('path' => '/etc/puppetlabs/enterprise/conf.d/pe.conf').always_return({ 'content' => '{}' }) expect(run_plan('peadm::upgrade', 'primary_host' => 'primary', @@ -50,7 +51,7 @@ def allow_standard_non_returning_calls expect_task('peadm::read_file') .with_params('path' => '/opt/puppetlabs/server/pe_build') .always_return({ 'content' => '2021.7.3' }) - + expect_task('peadm::read_file').with_params('path' => '/etc/puppetlabs/enterprise/conf.d/pe.conf').always_return({ 'content' => '{}' }) expect_task('peadm::cert_data').return_for_targets('primary' => trusted_primary, 'compiler' => trusted_compiler).be_called_times(1) expect_task('peadm::check_pe_master_rules').always_return(pe_rule_check).be_called_times(1) @@ -109,7 +110,7 @@ def allow_standard_non_returning_calls it 'updates pe.conf if r10k_known_hosts is set' do expect_task('peadm::read_file') .with_params('path' => '/etc/puppetlabs/enterprise/conf.d/pe.conf') - .always_return({ 'content' => <<~PECONF }) + .always_return({ 'content' => <<~PECONF }).be_called_times(2) # spec pe.conf "puppet_enterprise::puppet_master_host": "%{::trusted.certname}" PECONF @@ -132,6 +133,8 @@ def allow_standard_non_returning_calls expect_out_message.with_params(r10k_warning) expect_task('peadm::check_pe_master_rules').always_return(pe_rule_check) + expect_task('peadm::read_file').with_params('path' => '/etc/puppetlabs/enterprise/conf.d/pe.conf').always_return({ 'content' => '{}' }) + expect(run_plan('peadm::upgrade', 'primary_host' => 'primary', 'version' => '2023.3.0', @@ -145,6 +148,8 @@ def allow_standard_non_returning_calls expect_out_message.with_params(r10k_warning).not_be_called expect_task('peadm::check_pe_master_rules').always_return(pe_rule_check) + expect_task('peadm::read_file').with_params('path' => '/etc/puppetlabs/enterprise/conf.d/pe.conf').always_return({ 'content' => '{}' }) + expect(run_plan('peadm::upgrade', 'primary_host' => 'primary', 'version' => '2023.4.0',