diff --git a/documentation/install_and_configure_without_ha.md b/documentation/install_and_configure_without_ha.md new file mode 100644 index 00000000..610f6e0c --- /dev/null +++ b/documentation/install_and_configure_without_ha.md @@ -0,0 +1,64 @@ +# Install and configure Extra Large without HA + +* TODO: add this doc as a section to basic_usage.md instead? + +Please see the [basic_usage.md](basic_usage.md) document for reference; this document will avoid repeating the information covered there. +The install, configure, and upgrade plans covered in the [basic_usage.md](basic_usage.md) document can also set up the Extra Large environment without HA by omitting the optional settings `master_replica_host` and `puppetdb_database_replica_host` in the params.json file (see the [example](#example-params.json-bolt-parameters-file) below). + +## Basic usage instructions + +1. Ensure the hostname of each system is set correctly, to the same value that will be used to connect to the system, and refer to the system as. If the hostname is not set as expected the installation plan will refuse to continue. +2. Install Bolt on a jumphost. This can be the master, or any other system. +3. Download or git clone the pe\_xl module and put it somewhere on the jumphost, e.g. ~/modules/pe\_xl. +4. Create an inventory file with connection information. An example is included below. +5. Create a parameters file. An example is included below. Note the omission of the `master_replica_host` and `puppetdb_database_replica_host` parameters. +6. Run the pe\_xl plan with the inputs created. Example: +``` + bolt plan run pe_xl \ + --inventory nodes.yaml \ + --modulepath ~/modules \ + --params @params.json +``` + +### Example nodes.yaml Bolt inventory file + +```yaml + +--- +groups: + - name: pe_xl_nodes + config: + transport: ssh + ssh: + host-key-check: false + user: centos + run-as: root + tty: true + nodes: + - pe-xl-core-0.lab1.puppet.vm + - pe-xl-core-1.lab1.puppet.vm + - pe-xl-compiler-0.lab1.puppet.vm + - pe-xl-compiler-1.lab1.puppet.vm +``` + +### Example params.json Bolt parameters file + +```json +{ + "install": true, + "configure": true, + "upgrade": false, + + "master_host": "pe-xl-core-0.lab1.puppet.vm", + "puppetdb_database_host": "pe-xl-core-1.lab1.puppet.vm", + "compiler_hosts": [ + "pe-xl-compiler-0.lab1.puppet.vm", + "pe-xl-compiler-1.lab1.puppet.vm" + ], + + "console_password": "puppetlabs", + "dns_alt_names": [ "puppet", "puppet.lab1.puppet.vm" ], + "compiler_pool_address": "puppet.lab1.puppet.vm", + "version": "2019.1.0" +} +``` diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index e0791dad..e7cee9b8 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -50,14 +50,15 @@ # Because the group does not have any data by default this does not impact # out-of-box configuration of the group. node_group { 'PE Master': - parent => 'PE Infrastructure', - rule => ['or', + parent => 'PE Infrastructure', + rule => ['or', ['and', ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::compiler']], ['=', 'name', $master_host], ], - data => { + data => { 'pe_repo' => { 'compile_master_pool_address' => $compiler_pool_address }, }, + variables => { 'pe_master' => true }, } # This class has to be included here because puppet_enterprise is declared diff --git a/plans/configure.pp b/plans/configure.pp index 39cbbe72..8d544564 100644 --- a/plans/configure.pp +++ b/plans/configure.pp @@ -3,10 +3,11 @@ plan pe_xl::configure ( String[1] $master_host, String[1] $puppetdb_database_host, - String[1] $master_replica_host, - String[1] $puppetdb_database_replica_host, Array[String[1]] $compiler_hosts = [ ], + Optional[String[1]] $master_replica_host = undef, + Optional[String[1]] $puppetdb_database_replica_host = undef, + # This parameter exists primarily to enable the use case of running # pe_xl::configure over the PCP transport. An orchestrator restart happens # during provision replica. Running `bolt plan run` directly on the master @@ -21,6 +22,18 @@ String[1] $stagingdir = '/tmp', ) { + $ha_hosts = [ + $master_replica_host, + $puppetdb_database_replica_host, + ].pe_xl::flatten_compact() + + # Ensure valid input for HA + $ha = $ha_hosts.size ? { + 0 => false, + 2 => true, + default => fail("Must specify either both or neither of master_replica_host, puppetdb_database_replica_host"), + } + # Allow for the configure task to be run local to the master. $master_target = $executing_on_master ? { true => "local://${master_host}", @@ -53,7 +66,7 @@ # Run Puppet in no-op on the compilers so that their status in PuppetDB # is updated and they can be identified by the puppet_enterprise module as # CMs - run_task('pe_xl::puppet_runonce', [$compiler_hosts, $master_replica_host], + run_task('pe_xl::puppet_runonce', [$compiler_hosts, $master_replica_host].pe_xl::flatten_compact, noop => true, ) @@ -62,7 +75,7 @@ run_task('pe_xl::puppet_runonce', [ $puppetdb_database_host, $puppetdb_database_replica_host, - ]) + ].pe_xl::flatten_compact) # Run Puppet on the master to ensure all services configured and # running in prep for provisioning the replica. This is done separately so @@ -70,24 +83,28 @@ # other nodes to fail. run_task('pe_xl::puppet_runonce', $master_target) - # Run the PE Replica Provision - run_task('pe_xl::provision_replica', $master_target, - master_replica => $master_replica_host, - token_file => $token_file, - ) + if $ha { + # Run the PE Replica Provision + run_task('pe_xl::provision_replica', $master_target, + master_replica => $master_replica_host, + token_file => $token_file, + ) - # Run the PE Replica Enable - run_task('pe_xl::enable_replica', $master_target, - master_replica => $master_replica_host, - token_file => $token_file, - ) + # Run the PE Replica Enable + run_task('pe_xl::enable_replica', $master_target, + master_replica => $master_replica_host, + token_file => $token_file, + ) + } # Run Puppet everywhere to pick up last remaining config tweaks run_task('pe_xl::puppet_runonce', [ - $master_target, $master_replica_host, - $puppetdb_database_host, $puppetdb_database_replica_host, + $master_target, + $puppetdb_database_host, $compiler_hosts, - ].pe_xl::flatten_compact()) + $master_replica_host, + $puppetdb_database_replica_host, + ].pe_xl::flatten_compact) # Deploy an environment if a deploy environment is specified if $deploy_environment { diff --git a/plans/install.pp b/plans/install.pp index 13725c48..2f75219e 100644 --- a/plans/install.pp +++ b/plans/install.pp @@ -3,10 +3,11 @@ plan pe_xl::install ( String[1] $master_host, String[1] $puppetdb_database_host, - String[1] $master_replica_host, - String[1] $puppetdb_database_replica_host, Array[String[1]] $compiler_hosts = [ ], + Optional[String[1]] $master_replica_host = undef, + Optional[String[1]] $puppetdb_database_replica_host = undef, + String[1] $console_password, String[1] $version = '2018.1.3', Hash $r10k_sources = { }, @@ -16,19 +17,46 @@ ) { # Define a number of host groupings for use later in the plan - - $all_hosts = [ + $core_hosts = [ $master_host, $puppetdb_database_host, - $compiler_hosts, + ].pe_xl::flatten_compact() + + $ha_hosts = [ + $master_replica_host, + $puppetdb_database_replica_host, + ].pe_xl::flatten_compact() + + $ha_replica_target = [ $master_replica_host, + ].pe_xl::flatten_compact() + + $ha_database_target = [ + $puppetdb_database_replica_host, + ].pe_xl::flatten_compact() + + # Ensure valid input for HA + $ha = $ha_hosts.size ? { + 0 => false, + 2 => true, + default => fail("Must specify either both or neither of master_replica_host, puppetdb_database_replica_host"), + } + + $all_hosts = [ + $core_hosts, + $ha_hosts, + $compiler_hosts, + ].pe_xl::flatten_compact() + + $database_hosts = [ + $puppetdb_database_host, $puppetdb_database_replica_host, ].pe_xl::flatten_compact() $pe_installer_hosts = [ $master_host, $puppetdb_database_host, - $master_replica_host, + $puppetdb_database_replica_host, ].pe_xl::flatten_compact() $agent_installer_hosts = [ @@ -43,8 +71,14 @@ $pp_role = '1.3.6.1.4.1.34380.1.1.13' # Clusters A and B are used to divide PuppetDB availability for compilers - $cm_cluster_a = $compiler_hosts.filter |$index,$cm| { $index % 2 == 0 } - $cm_cluster_b = $compiler_hosts.filter |$index,$cm| { $index % 2 != 0 } + if $ha { + $cm_cluster_a = $compiler_hosts.filter |$index,$cm| { $index % 2 == 0 } + $cm_cluster_b = $compiler_hosts.filter |$index,$cm| { $index % 2 != 0 } + } + else { + $cm_cluster_a = $compiler_hosts + $cm_cluster_b = [] + } $dns_alt_names_csv = $dns_alt_names.reduce |$csv,$x| { "${csv},${x}" } @@ -78,7 +112,7 @@ # Upload the pe.conf files to the hosts that need them pe_xl::file_content_upload($master_pe_conf, '/tmp/pe.conf', $master_host) pe_xl::file_content_upload($puppetdb_database_pe_conf, '/tmp/pe.conf', $puppetdb_database_host) - pe_xl::file_content_upload($puppetdb_database_replica_pe_conf, '/tmp/pe.conf', $puppetdb_database_replica_host) + pe_xl::file_content_upload($puppetdb_database_replica_pe_conf, '/tmp/pe.conf', $ha_database_target) # Download the PE tarball and send it to the nodes that need it $pe_tarball_name = "puppet-enterprise-${version}-el-7-x86_64.tar.gz" @@ -86,7 +120,7 @@ $upload_tarball_path = "/tmp/${pe_tarball_name}" run_plan('pe_xl::util::retrieve_and_upload', - nodes => [$master_host, $puppetdb_database_host, $puppetdb_database_replica_host], + nodes => $pe_installer_hosts, source => "https://s3.amazonaws.com/pe-builds/released/${version}/puppet-enterprise-${version}-el-7-x86_64.tar.gz", local_path => $local_tarball_path, upload_path => $upload_tarball_path, @@ -115,7 +149,7 @@ | HEREDOC ) - run_task('pe_xl::mkdir_p_file', $puppetdb_database_replica_host, + run_task('pe_xl::mkdir_p_file', $ha_database_target, path => '/etc/puppetlabs/puppet/csr_attributes.yaml', content => @("HEREDOC"), --- @@ -129,14 +163,14 @@ # Get the master installation up and running. The installer will # "fail" because PuppetDB can't start. That's expected. without_default_logging() || { - notice("Starting: task pe_xl::pe_install on ${master_host}") + out::message("Starting: task pe_xl::pe_install on ${master_host}") run_task('pe_xl::pe_install', $master_host, _catch_errors => true, tarball => $upload_tarball_path, peconf => '/tmp/pe.conf', shortcircuit_puppetdb => true, ) - notice("Finished: task pe_xl::pe_install on ${master_host}") + out::message("Finished: task pe_xl::pe_install on ${master_host}") } # Configure autosigning for the puppetdb database hosts 'cause they need it @@ -145,14 +179,11 @@ owner => 'pe-puppet', group => 'pe-puppet', mode => '0644', - content => @("HEREDOC"), - ${puppetdb_database_host} - ${puppetdb_database_replica_host} - | HEREDOC + content => $database_hosts.reduce |$memo,$host| { "${host}\n${memo}" }, ) # Run the PE installer on the puppetdb database hosts - run_task('pe_xl::pe_install', [$puppetdb_database_host, $puppetdb_database_replica_host], + run_task('pe_xl::pe_install', $database_hosts, tarball => $upload_tarball_path, peconf => '/tmp/pe.conf', ) @@ -184,7 +215,7 @@ ) # Deploy the PE agent to all remaining hosts - run_task('pe_xl::agent_install', $master_replica_host, + run_task('pe_xl::agent_install', $ha_replica_target, server => $master_host, install_flags => [ '--puppet-service-ensure', 'stopped', @@ -220,9 +251,9 @@ # Do a Puppet agent run to ensure certificate requests have been submitted # These runs will "fail", and that's expected. without_default_logging() || { - notice("Starting: task pe_xl::puppet_runonce on ${agent_installer_hosts}") + out::message("Starting: task pe_xl::puppet_runonce on ${agent_installer_hosts}") run_task('pe_xl::puppet_runonce', $agent_installer_hosts, {_catch_errors => true}) - notice("Finished: task pe_xl::puppet_runonce on ${agent_installer_hosts}") + out::message("Finished: task pe_xl::puppet_runonce on ${agent_installer_hosts}") } # Ensure some basic configuration on the master needed at install time. diff --git a/plans/upgrade.pp b/plans/upgrade.pp index fad0b218..0772566e 100644 --- a/plans/upgrade.pp +++ b/plans/upgrade.pp @@ -3,8 +3,8 @@ plan pe_xl::upgrade ( String[1] $master_host, String[1] $puppetdb_database_host, - String[1] $master_replica_host, - String[1] $puppetdb_database_replica_host, + Optional[String[1]] $master_replica_host = undef, + Optional[String[1]] $puppetdb_database_replica_host = undef, String[1] $version = '2018.1.4', @@ -12,6 +12,14 @@ String[1] $pe_source = "https://s3.amazonaws.com/pe-builds/released/${version}/puppet-enterprise-${version}-el-7-x86_64.tar.gz", ) { + $ha_replica_target = [ + $master_replica_host, + ].pe_xl::flatten_compact() + + $ha_database_target = [ + $puppetdb_database_replica_host, + ].pe_xl::flatten_compact() + # Look up which hosts are compilers in the stack # We look up groups of CMs separately since when they are upgraded is determined # by which PDB PG host they are affiliated with @@ -47,11 +55,13 @@ # Download the PE tarball on the nodes that need it $upload_tarball_path = "/tmp/puppet-enterprise-${version}-el-7-x86_64.tar.gz" - run_task('pe_xl::download', [ - $master_host, - $puppetdb_database_host, - $puppetdb_database_replica_host - ], + $download_hosts = [ + $master_host, + $puppetdb_database_host, + $puppetdb_database_replica_host, + ].pe_xl::flatten_compact() + + run_task('pe_xl::download', $download_hosts, source => $pe_source, path => $upload_tarball_path, ) @@ -124,15 +134,15 @@ ) # Run the upgrade.sh script on the master replica host - run_task('pe_xl::agent_upgrade', $master_replica_host, + run_task('pe_xl::agent_upgrade', $ha_replica_target, server => $master_host, ) # Upgrade the master replica's PuppetDB PostgreSQL host - run_task('pe_xl::pe_install', $puppetdb_database_replica_host, + run_task('pe_xl::pe_install', $ha_database_target, tarball => $upload_tarball_path, ) - run_task('pe_xl::puppet_runonce', $puppetdb_database_replica_host) + run_task('pe_xl::puppet_runonce', $ha_database_target) # Upgrade the compiler group B hosts run_task('pe_xl::agent_upgrade', $compiler_cluster_master_replica_hosts, diff --git a/tasks/configure_node_groups.sh b/tasks/configure_node_groups.sh index 0a1a9662..5878f3e4 100755 --- a/tasks/configure_node_groups.sh +++ b/tasks/configure_node_groups.sh @@ -3,7 +3,12 @@ /opt/puppetlabs/bin/puppet apply --environment production <<'EOF' -function param($name) { inline_template("<%= ENV['PT_${name}'] %>") } +function param($name) { + ($var = inline_template("<%= ENV['PT_${name}'] %>")) ? { + '' => undef, + default => $var, + } +} class configure_node_groups ( String[1] $master_host = param('master_host'), @@ -44,14 +49,15 @@ class configure_node_groups ( # Because the group does not have any data by default this does not impact # out-of-box configuration of the group. node_group { 'PE Master': - parent => 'PE Infrastructure', - rule => ['or', + parent => 'PE Infrastructure', + rule => ['or', ['and', ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::compiler']], ['=', 'name', $master_host], ], - data => { + data => { 'pe_repo' => { 'compile_master_pool_address' => $compiler_pool_address }, }, + variables => { 'pe_master' => true }, } # This class has to be included here because puppet_enterprise is declared @@ -162,3 +168,6 @@ class configure_node_groups ( } } + +include configure_node_groups +EOF diff --git a/templates/puppetdb_database-pe.conf.epp b/templates/puppetdb_database-pe.conf.epp index 53036367..deab9e69 100644 --- a/templates/puppetdb_database-pe.conf.epp +++ b/templates/puppetdb_database-pe.conf.epp @@ -1,5 +1,5 @@ <%- | String[1] $master_host, - String[1] $puppetdb_database_host, + Optional[String[1]] $puppetdb_database_host, | -%> #---------------------------------------------------------------------------- # Puppet Enterprise installer configuration file