(SLV-365) Update the install, configure, and upgrade plans in the pe_xl module to make ha optional

documentation/install_and_configure_without_ha.md

@@ -0,0 +1,64 @@
+# Install and configure Extra Large without HA
+
+* TODO: add this doc as a section to basic_usage.md instead?
+
+Please see the [basic_usage.md](basic_usage.md) document for reference; this document will avoid repeating the information covered there.
+The install, configure, and upgrade plans covered in the [basic_usage.md](basic_usage.md) document can also set up the Extra Large environment without HA by omitting the optional settings `master_replica_host` and `puppetdb_database_replica_host` in the params.json file (see the [example](#example-params.json-bolt-parameters-file) below).
+
+## Basic usage instructions
+
+1. Ensure the hostname of each system is set correctly, to the same value that will be used to connect to the system, and refer to the system as. If the hostname is not set as expected the installation plan will refuse to continue.
+2. Install Bolt on a jumphost. This can be the master, or any other system.
+3. Download or git clone the pe\_xl module and put it somewhere on the jumphost, e.g. ~/modules/pe\_xl.
+4. Create an inventory file with connection information. An example is included below.
+5. Create a parameters file. An example is included below. Note the omission of the `master_replica_host` and `puppetdb_database_replica_host` parameters.
+6. Run the pe\_xl plan with the inputs created. Example:
+```
+        bolt plan run pe_xl \
+          --inventory nodes.yaml \
+          --modulepath ~/modules \
+          --params @params.json 
+```
+
+### Example nodes.yaml Bolt inventory file
+
+```yaml
+
+---
+groups:
+  - name: pe_xl_nodes
+    config:
+      transport: ssh
+      ssh:
+        host-key-check: false
+        user: centos
+        run-as: root
+        tty: true
+    nodes:
+      - pe-xl-core-0.lab1.puppet.vm
+      - pe-xl-core-1.lab1.puppet.vm
+      - pe-xl-compiler-0.lab1.puppet.vm
+      - pe-xl-compiler-1.lab1.puppet.vm
+```
+
+### Example params.json Bolt parameters file
+
+```json
+{
+  "install": true,
+  "configure": true,
+  "upgrade": false,
+
+  "master_host": "pe-xl-core-0.lab1.puppet.vm",
+  "puppetdb_database_host": "pe-xl-core-1.lab1.puppet.vm",
+  "compiler_hosts": [
+    "pe-xl-compiler-0.lab1.puppet.vm",
+    "pe-xl-compiler-1.lab1.puppet.vm"
+  ],
+
+  "console_password": "puppetlabs",
+  "dns_alt_names": [ "puppet", "puppet.lab1.puppet.vm" ],
+  "compiler_pool_address": "puppet.lab1.puppet.vm",
+  "version": "2019.1.0"
+}
+```

manifests/setup/node_manager.pp

@@ -50,14 +50,15 @@
   # Because the group does not have any data by default this does not impact
   # out-of-box configuration of the group.
   node_group { 'PE Master':
-    parent => 'PE Infrastructure',
-    rule   => ['or',
+    parent    => 'PE Infrastructure',
+    rule      => ['or',
       ['and', ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::compiler']],
       ['=', 'name', $master_host],
     ],
-    data   => {
+    data      => {
       'pe_repo' => { 'compile_master_pool_address' => $compiler_pool_address },
     },
+    variables => { 'pe_master' => true },
   }
 
   # This class has to be included here because puppet_enterprise is declared

plans/configure.pp

@@ -3,10 +3,11 @@
 plan pe_xl::configure (
   String[1]           $master_host,
   String[1]           $puppetdb_database_host,
-  String[1]           $master_replica_host,
-  String[1]           $puppetdb_database_replica_host,
   Array[String[1]]    $compiler_hosts = [ ],
 
+  Optional[String[1]] $master_replica_host = undef,
+  Optional[String[1]] $puppetdb_database_replica_host = undef,
+
   # This parameter exists primarily to enable the use case of running
   # pe_xl::configure over the PCP transport. An orchestrator restart happens
   # during provision replica. Running `bolt plan run` directly on the master
@@ -21,6 +22,18 @@
   String[1]           $stagingdir = '/tmp',
 ) {
 
+  $ha_hosts = [
+    $master_replica_host,
+    $puppetdb_database_replica_host,
+  ].pe_xl::flatten_compact()
+
+  # Ensure valid input for HA
+  $ha = $ha_hosts.size ? {
+    0       => false,
+    2       => true,
+    default => fail("Must specify either both or neither of master_replica_host, puppetdb_database_replica_host"),
+  }
+
   # Allow for the configure task to be run local to the master.
   $master_target = $executing_on_master ? {
     true  => "local://${master_host}",
@@ -53,7 +66,7 @@
   # Run Puppet in no-op on the compilers so that their status in PuppetDB
   # is updated and they can be identified by the puppet_enterprise module as
   # CMs
-  run_task('pe_xl::puppet_runonce', [$compiler_hosts, $master_replica_host],
+  run_task('pe_xl::puppet_runonce', [$compiler_hosts, $master_replica_host].pe_xl::flatten_compact,
     noop => true,
   )
 
@@ -62,32 +75,36 @@
   run_task('pe_xl::puppet_runonce', [
     $puppetdb_database_host,
     $puppetdb_database_replica_host,
-  ])
+  ].pe_xl::flatten_compact)
 
   # Run Puppet on the master to ensure all services configured and
   # running in prep for provisioning the replica. This is done separately so
   # that a service restart of pe-puppetserver doesn't cause Puppet runs on
   # other nodes to fail.
   run_task('pe_xl::puppet_runonce', $master_target)
 
-  # Run the PE Replica Provision
-  run_task('pe_xl::provision_replica', $master_target,
-    master_replica         => $master_replica_host,
-    token_file             => $token_file,
-  )
+  if $ha {
+    # Run the PE Replica Provision
+    run_task('pe_xl::provision_replica', $master_target,
+      master_replica         => $master_replica_host,
+      token_file             => $token_file,
+    )
 
-  # Run the PE Replica Enable
-  run_task('pe_xl::enable_replica', $master_target,
-    master_replica         => $master_replica_host,
-    token_file             => $token_file,
-  )
+    # Run the PE Replica Enable
+    run_task('pe_xl::enable_replica', $master_target,
+      master_replica         => $master_replica_host,
+      token_file             => $token_file,
+    )
+  }
 
   # Run Puppet everywhere to pick up last remaining config tweaks
   run_task('pe_xl::puppet_runonce', [
-    $master_target, $master_replica_host,
-    $puppetdb_database_host, $puppetdb_database_replica_host,
+    $master_target,
+    $puppetdb_database_host,
     $compiler_hosts,
-  ].pe_xl::flatten_compact())
+    $master_replica_host,
+    $puppetdb_database_replica_host,
+  ].pe_xl::flatten_compact)
 
   # Deploy an environment if a deploy environment is specified
   if $deploy_environment {

plans/install.pp

@@ -3,10 +3,11 @@
 plan pe_xl::install (
   String[1]           $master_host,
   String[1]           $puppetdb_database_host,
-  String[1]           $master_replica_host,
-  String[1]           $puppetdb_database_replica_host,
   Array[String[1]]    $compiler_hosts = [ ],
 
+  Optional[String[1]] $master_replica_host = undef,
+  Optional[String[1]] $puppetdb_database_replica_host = undef,
+
   String[1]           $console_password,
   String[1]           $version = '2018.1.3',
   Hash                $r10k_sources = { },
@@ -16,19 +17,46 @@
 ) {
 
   # Define a number of host groupings for use later in the plan
-
-  $all_hosts = [
+  $core_hosts = [
     $master_host,
     $puppetdb_database_host,
-    $compiler_hosts,
+  ].pe_xl::flatten_compact()
+
+  $ha_hosts = [
+    $master_replica_host,
+    $puppetdb_database_replica_host,
+  ].pe_xl::flatten_compact()
+
+  $ha_replica_target = [
     $master_replica_host,
+  ].pe_xl::flatten_compact()
+
+  $ha_database_target = [
+    $puppetdb_database_replica_host,
+  ].pe_xl::flatten_compact()
+
+  # Ensure valid input for HA
+  $ha = $ha_hosts.size ? {
+    0       => false,
+    2       => true,
+    default => fail("Must specify either both or neither of master_replica_host, puppetdb_database_replica_host"),
+  }
+
+  $all_hosts = [
+    $core_hosts,
+    $ha_hosts,
+    $compiler_hosts,
+  ].pe_xl::flatten_compact()
+
+  $database_hosts = [
+    $puppetdb_database_host,
     $puppetdb_database_replica_host,
   ].pe_xl::flatten_compact()
 
   $pe_installer_hosts = [
     $master_host,
     $puppetdb_database_host,
-    $master_replica_host,
+    $puppetdb_database_replica_host,
   ].pe_xl::flatten_compact()
 
   $agent_installer_hosts = [
@@ -43,8 +71,14 @@
   $pp_role        = '1.3.6.1.4.1.34380.1.1.13'
 
   # Clusters A and B are used to divide PuppetDB availability for compilers
-  $cm_cluster_a = $compiler_hosts.filter |$index,$cm| { $index % 2 == 0 }
-  $cm_cluster_b = $compiler_hosts.filter |$index,$cm| { $index % 2 != 0 }
+  if $ha {
+    $cm_cluster_a = $compiler_hosts.filter |$index,$cm| { $index % 2 == 0 }
+    $cm_cluster_b = $compiler_hosts.filter |$index,$cm| { $index % 2 != 0 }
+  }
+  else {
+    $cm_cluster_a = $compiler_hosts
+    $cm_cluster_b = []
+  }
 
   $dns_alt_names_csv = $dns_alt_names.reduce |$csv,$x| { "${csv},${x}" }
 
@@ -78,15 +112,15 @@
   # Upload the pe.conf files to the hosts that need them
   pe_xl::file_content_upload($master_pe_conf, '/tmp/pe.conf', $master_host)
   pe_xl::file_content_upload($puppetdb_database_pe_conf, '/tmp/pe.conf', $puppetdb_database_host)
-  pe_xl::file_content_upload($puppetdb_database_replica_pe_conf, '/tmp/pe.conf', $puppetdb_database_replica_host)
+  pe_xl::file_content_upload($puppetdb_database_replica_pe_conf, '/tmp/pe.conf', $ha_database_target)
 
   # Download the PE tarball and send it to the nodes that need it
   $pe_tarball_name     = "puppet-enterprise-${version}-el-7-x86_64.tar.gz"
   $local_tarball_path  = "${stagingdir}/${pe_tarball_name}"
   $upload_tarball_path = "/tmp/${pe_tarball_name}"
 
   run_plan('pe_xl::util::retrieve_and_upload',
-    nodes       => [$master_host, $puppetdb_database_host, $puppetdb_database_replica_host],
+    nodes       => $pe_installer_hosts,
     source      => "https://s3.amazonaws.com/pe-builds/released/${version}/puppet-enterprise-${version}-el-7-x86_64.tar.gz",
     local_path  => $local_tarball_path,
     upload_path => $upload_tarball_path,
@@ -115,7 +149,7 @@
       | HEREDOC
   )
 
-  run_task('pe_xl::mkdir_p_file', $puppetdb_database_replica_host,
+  run_task('pe_xl::mkdir_p_file', $ha_database_target,
     path    => '/etc/puppetlabs/puppet/csr_attributes.yaml',
     content => @("HEREDOC"),
       ---
@@ -129,14 +163,14 @@
   # Get the master installation up and running. The installer will
   # "fail" because PuppetDB can't start. That's expected.
   without_default_logging() || {
-    notice("Starting: task pe_xl::pe_install on ${master_host}")
+    out::message("Starting: task pe_xl::pe_install on ${master_host}")
     run_task('pe_xl::pe_install', $master_host,
       _catch_errors         => true,
       tarball               => $upload_tarball_path,
       peconf                => '/tmp/pe.conf',
       shortcircuit_puppetdb => true,
     )
-    notice("Finished: task pe_xl::pe_install on ${master_host}")
+    out::message("Finished: task pe_xl::pe_install on ${master_host}")
   }
 
   # Configure autosigning for the puppetdb database hosts 'cause they need it
@@ -145,14 +179,11 @@
     owner   => 'pe-puppet',
     group   => 'pe-puppet',
     mode    => '0644',
-    content => @("HEREDOC"),
-      ${puppetdb_database_host}
-      ${puppetdb_database_replica_host}
-      | HEREDOC
+    content => $database_hosts.reduce |$memo,$host| { "${host}\n${memo}" },
   )
 
   # Run the PE installer on the puppetdb database hosts
-  run_task('pe_xl::pe_install', [$puppetdb_database_host, $puppetdb_database_replica_host],
+  run_task('pe_xl::pe_install', $database_hosts,
     tarball => $upload_tarball_path,
     peconf  => '/tmp/pe.conf',
   )
@@ -184,7 +215,7 @@
   )
 
   # Deploy the PE agent to all remaining hosts
-  run_task('pe_xl::agent_install', $master_replica_host,
+  run_task('pe_xl::agent_install', $ha_replica_target,
     server        => $master_host,
     install_flags => [
       '--puppet-service-ensure', 'stopped',
@@ -220,9 +251,9 @@
   # Do a Puppet agent run to ensure certificate requests have been submitted
   # These runs will "fail", and that's expected.
   without_default_logging() || {
-    notice("Starting: task pe_xl::puppet_runonce on ${agent_installer_hosts}")
+    out::message("Starting: task pe_xl::puppet_runonce on ${agent_installer_hosts}")
     run_task('pe_xl::puppet_runonce', $agent_installer_hosts, {_catch_errors => true})
-    notice("Finished: task pe_xl::puppet_runonce on ${agent_installer_hosts}")
+    out::message("Finished: task pe_xl::puppet_runonce on ${agent_installer_hosts}")
   }
 
   # Ensure some basic configuration on the master needed at install time.