From 3866ba0f4a2cc8afa1212a52aae3febe37c3b269 Mon Sep 17 00:00:00 2001 From: Reid Vandewiele Date: Fri, 8 Nov 2019 18:12:34 -0800 Subject: [PATCH] Switch to using apply blocks for node_manager It took awhile to figure out how to do this, but I think this latest iteration actually works. --- manifests/setup/node_manager.pp | 27 ++--- plans/test.pp | 33 ------ plans/unit/configure.pp | 41 +++---- plans/util/install_module.pp | 29 ----- tasks/configure_node_groups.json | 29 ----- tasks/configure_node_groups.sh | 176 ------------------------------- 6 files changed, 36 insertions(+), 299 deletions(-) delete mode 100644 plans/test.pp delete mode 100644 plans/util/install_module.pp delete mode 100644 tasks/configure_node_groups.json delete mode 100755 tasks/configure_node_groups.sh diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index e7cee9b8..a2fd5a15 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -61,18 +61,21 @@ variables => { 'pe_master' => true }, } - # This class has to be included here because puppet_enterprise is declared - # in the console with parameters. It is therefore not possible to include - # puppet_enterprise::profile::database in code without causing a conflict. - node_group { 'PE Database': - ensure => present, - parent => 'PE Infrastructure', - environment => 'production', - override_environment => false, - rule => ['and', ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::puppetdb_database']], - classes => { - 'puppet_enterprise::profile::database' => { }, - }, + # Create the database group if a database host is external + if ($puppetdb_database_host != $master_host) { + # This class has to be included here because puppet_enterprise is declared + # in the console with parameters. It is therefore not possible to include + # puppet_enterprise::profile::database in code without causing a conflict. + node_group { 'PE Database': + ensure => present, + parent => 'PE Infrastructure', + environment => 'production', + override_environment => false, + rule => ['and', ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::puppetdb_database']], + classes => { + 'puppet_enterprise::profile::database' => { }, + }, + } } # Create data-only groups to store PuppetDB PostgreSQL database configuration diff --git a/plans/test.pp b/plans/test.pp deleted file mode 100644 index 37e9086c..00000000 --- a/plans/test.pp +++ /dev/null @@ -1,33 +0,0 @@ -plan pe_xl::test ( - TargetSpec $nodes, -) { - $target = get_target($nodes) - - $servername = $target.host - apply($target) { - - file { 'node_manager.yaml': - ensure => file, - mode => '0644', - path => Deferred('pe_xl::node_manager_yaml_location'), - content => epp('pe_xl/node_manager.yaml.epp', { - server => $servername, - }), - } - - Node_group { - require => File['node_manager.yaml'], - } - - node_group { 'Test Group': - ensure => present, - parent => 'PE Infrastructure', - rule => ['and', - ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::master'], - ['=', ['trusted', 'extensions', 'pp_cluster'], 'B'], - ], - } - - } - -} diff --git a/plans/unit/configure.pp b/plans/unit/configure.pp index 6a1a8f95..54a3b4f8 100644 --- a/plans/unit/configure.pp +++ b/plans/unit/configure.pp @@ -37,28 +37,29 @@ $compiler_hosts, ) - # Retrieve and deploy Puppet modules from the Forge so that they can be used - # for ensuring some configuration (node groups) - [ ['WhatsARanjit-node_manager', '0.7.1'], - ['puppetlabs-stdlib', '5.0.0'], - ].each |$tuple| { - run_plan('pe_xl::util::install_module', - nodes => $master_target, - module => $tuple[0], - version => $tuple[1], - stagingdir => $stagingdir, - ) - } - # Set up the console node groups to configure the various hosts in their # roles - run_task('pe_xl::configure_node_groups', $master_target, - master_host => $master_target.pe_xl::target_host(), - master_replica_host => $master_replica_target.pe_xl::target_host(), - puppetdb_database_host => $puppetdb_database_target.pe_xl::target_host(), - puppetdb_database_replica_host => $puppetdb_database_replica_target.pe_xl::target_host(), - compiler_pool_address => $compiler_pool_address, - ) + apply($master_target) { + # Necessary to give the sandboxed Puppet executor the configuration + # necessary to connect to the classifier` + file { 'node_manager.yaml': + ensure => file, + mode => '0644', + path => Deferred('pe_xl::node_manager_yaml_location'), + content => epp('pe_xl/node_manager.yaml.epp', { + server => $servername, + }), + } + + class { 'pe_xl::configure_node_groups': + master_host => $master_target.pe_xl::target_host(), + master_replica_host => $master_replica_target.pe_xl::target_host(), + puppetdb_database_host => $puppetdb_database_target.pe_xl::target_host(), + puppetdb_database_replica_host => $puppetdb_database_replica_target.pe_xl::target_host(), + compiler_pool_address => $compiler_pool_address, + require => File['node_manager.yaml'], + } + } # Run Puppet in no-op on the compilers so that their status in PuppetDB # is updated and they can be identified by the puppet_enterprise module as diff --git a/plans/util/install_module.pp b/plans/util/install_module.pp deleted file mode 100644 index e33068d2..00000000 --- a/plans/util/install_module.pp +++ /dev/null @@ -1,29 +0,0 @@ -plan pe_xl::util::install_module( - TargetSpec $nodes, - String[1] $module, - String[1] $version, - String[1] $stagingdir = '/tmp', -) { - - $module_tarball = "${module.regsubst('/', '-')}-${version}.tar.gz" - - run_plan('pe_xl::util::retrieve_and_upload', - nodes => $nodes, - source => "https://forge.puppet.com/v3/files/${module_tarball}", - local_path => "${stagingdir}/${module_tarball}", - upload_path => "/tmp/${module_tarball}", - ) - - run_command(@("HEREDOC"), $nodes) - /opt/puppetlabs/bin/puppet module install \ - --modulepath /etc/puppetlabs/code-staging/environments/production/modules \ - --ignore-dependencies \ - /tmp/${module_tarball} - | HEREDOC - - run_command('chown -R pe-puppet:pe-puppet /etc/puppetlabs/code-staging', $nodes) - run_task('pe_xl::code_manager', $nodes, - action => 'commit', - ) - -} diff --git a/tasks/configure_node_groups.json b/tasks/configure_node_groups.json deleted file mode 100644 index ba8dcece..00000000 --- a/tasks/configure_node_groups.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "description": "Configure console node groups for a new install", - "parameters": { - "master_host": { - "type": "String[1]", - "description": "The certname of the master" - }, - "compiler_pool_address": { - "type": "String[1]", - "description": "The service name to use for the compiler pool" - }, - "puppetdb_database_host": { - "type": "Optional[String[1]]", - "description": "The certname of the PuppetDB database" - }, - "master_replica_host": { - "type": "Optional[String[1]]", - "description": "The certname of the master replica" - }, - "puppetdb_database_replica_host": { - "type": "Optional[String[1]]", - "description": "The certname of the PuppetDB database replica" - } - }, - "input_method": "environment", - "implementations": [ - {"name": "configure_node_groups.sh"} - ] -} diff --git a/tasks/configure_node_groups.sh b/tasks/configure_node_groups.sh deleted file mode 100755 index d712f512..00000000 --- a/tasks/configure_node_groups.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/bin/bash - - -/opt/puppetlabs/bin/puppet apply --environment production <<'EOF' - -function param($name) { - ($var = inline_template("<%= ENV['PT_${name}'] %>")) ? { - '' => undef, - default => $var, - } -} - -class configure_node_groups ( - String[1] $master_host = param('master_host'), - String[1] $puppetdb_database_host = param('puppetdb_database_host'), - String[1] $compiler_pool_address = param('compiler_pool_address'), - - Optional[String[1]] $master_replica_host = param('master_replica_host'), - Optional[String[1]] $puppetdb_database_replica_host = param('puppetdb_database_replica_host'), -) { - - if ([$master_replica_host, $puppetdb_database_replica_host].filter |$_| { $_ }.size == 1) { - fail('Must pass both master_replica_host and puppetdb_database_replica_host, or neither') - } - - ################################################## - # PE INFRASTRUCTURE GROUPS - ################################################## - - # Hiera data tuning for compilers - $compiler_data = { - 'puppet_enterprise::profile::puppetdb' => { - 'gc_interval' => '0', - }, - 'puppet_enterprise::puppetdb' => { - 'command_processing_threads' => 2, - 'write_maximum_pool_size' => 4, - 'read_maximum_pool_size' => 10, - }, - } - - # We modify this group's rule such that all PE infrastructure nodes will be - # members. - node_group { 'PE Infrastructure Agent': - rule => ['and', ['~', ['trusted', 'extensions', 'pp_role'], '^pe_xl::']], - } - - # We modify this group to add, as data, the compiler_pool_address only. - # Because the group does not have any data by default this does not impact - # out-of-box configuration of the group. - node_group { 'PE Master': - parent => 'PE Infrastructure', - rule => ['or', - ['and', ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::compiler']], - ['=', 'name', $master_host], - ], - data => { - 'pe_repo' => { 'compile_master_pool_address' => $compiler_pool_address }, - }, - variables => { 'pe_master' => true }, - } - - # Create the database group if a database host is external - if ($puppetdb_database_host != $master_host) { - # This class has to be included here because puppet_enterprise is declared - # in the console with parameters. It is therefore not possible to include - # puppet_enterprise::profile::database in code without causing a conflict. - node_group { 'PE Database': - ensure => present, - parent => 'PE Infrastructure', - environment => 'production', - override_environment => false, - rule => ['and', ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::puppetdb_database']], - classes => { - 'puppet_enterprise::profile::database' => { }, - }, - } - } - - # Create data-only groups to store PuppetDB PostgreSQL database configuration - # information specific to the master and master replica nodes. - node_group { 'PE Master A': - ensure => present, - parent => 'PE Infrastructure', - rule => ['and', - ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::master'], - ['=', ['trusted', 'extensions', 'pp_cluster'], 'A'], - ], - data => { - 'puppet_enterprise::profile::primary_master_replica' => { - 'database_host_puppetdb' => $puppetdb_database_host, - }, - 'puppet_enterprise::profile::puppetdb' => { - 'database_host' => $puppetdb_database_host, - }, - }, - } - - # Configure the A pool for compilers. There are up to two pools for HA, each - # having an affinity for one "availability zone" or the other. - node_group { 'PE Compiler Group A': - ensure => 'present', - parent => 'PE Master', - rule => ['and', - ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::compiler'], - ['=', ['trusted', 'extensions', 'pp_cluster'], 'A'], - ], - classes => { - 'puppet_enterprise::profile::puppetdb' => { - 'database_host' => $puppetdb_database_host, - }, - 'puppet_enterprise::profile::master' => { - 'puppetdb_host' => ['${clientcert}', $master_replica_host].filter |$_| { $_ }, # lint:ignore:single_quote_string_with_variables - 'puppetdb_port' => [8081], - } - }, - data => $compiler_data, - } - - # Create the replica and B groups if a replica master and database host are - # supplied - if ($master_replica_host and $puppetdb_database_replica_host) { - # We need to pre-create this group so that the master replica can be - # identified as running PuppetDB, so that Puppet will create a pg_ident - # authorization rule for it on the PostgreSQL nodes. - node_group { 'PE HA Replica': - ensure => 'present', - parent => 'PE Infrastructure', - rule => ['or', ['=', 'name', $master_replica_host]], - classes => { - 'puppet_enterprise::profile::primary_master_replica' => { } - }, - variables => { 'pe_xl_replica' => true }, - } - - node_group { 'PE Master B': - ensure => present, - parent => 'PE Infrastructure', - rule => ['and', - ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::master'], - ['=', ['trusted', 'extensions', 'pp_cluster'], 'B'], - ], - data => { - 'puppet_enterprise::profile::primary_master_replica' => { - 'database_host_puppetdb' => $puppetdb_database_replica_host, - }, - 'puppet_enterprise::profile::puppetdb' => { - 'database_host' => $puppetdb_database_replica_host, - }, - }, - } - - node_group { 'PE Compiler Group B': - ensure => 'present', - parent => 'PE Master', - rule => ['and', - ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::compiler'], - ['=', ['trusted', 'extensions', 'pp_cluster'], 'B'], - ], - classes => { - 'puppet_enterprise::profile::puppetdb' => { - 'database_host' => $puppetdb_database_replica_host, - }, - 'puppet_enterprise::profile::master' => { - 'puppetdb_host' => ['${clientcert}', $master_host], # lint:ignore:single_quote_string_with_variables - 'puppetdb_port' => [8081], - } - }, - data => $compiler_data, - } - } - -} - -include configure_node_groups -EOF