From 6a5047109c235e74ca01fb44ae3ded55c48e8523 Mon Sep 17 00:00:00 2001
From: "bill.claytor" <bill.claytor@puppet.com>
Date: Wed, 10 Jul 2019 12:05:29 -0700
Subject: [PATCH 01/15] (SLV-365) Update pe_xl plans to make ha optional

---
 plans/configure.pp | 144 +++++++++++++++++----
 plans/init.pp      |   4 +
 plans/install.pp   | 315 ++++++++++++++++++++++++++++++++++++++-------
 plans/upgrade.pp   | 146 ++++++++++++++++-----
 4 files changed, 497 insertions(+), 112 deletions(-)

diff --git a/plans/configure.pp b/plans/configure.pp
index 39cbbe72..eb44e981 100644
--- a/plans/configure.pp
+++ b/plans/configure.pp
@@ -1,6 +1,7 @@
 # @summary Configure first-time classification and HA setup
 #
 plan pe_xl::configure (
+  Boolean             $ha,
   String[1]           $master_host,
   String[1]           $puppetdb_database_host,
   String[1]           $master_replica_host,
@@ -21,6 +22,8 @@
   String[1]           $stagingdir = '/tmp',
 ) {
 
+  # TODO: remove 'SLV-365' comments
+
   # Allow for the configure task to be run local to the master.
   $master_target = $executing_on_master ? {
     true  => "local://${master_host}",
@@ -42,27 +45,76 @@
 
   # Set up the console node groups to configure the various hosts in their
   # roles
-  run_task('pe_xl::configure_node_groups', $master_target,
-    master_host                    => $master_host,
-    master_replica_host            => $master_replica_host,
-    puppetdb_database_host         => $puppetdb_database_host,
-    puppetdb_database_replica_host => $puppetdb_database_replica_host,
-    compiler_pool_address          => $compiler_pool_address,
-  )
+
+  # SLV-365
+  # run_task('pe_xl::configure_node_groups', $master_target,
+  #   master_host                    => $master_host,
+  #   master_replica_host            => $master_replica_host,
+  #   puppetdb_database_host         => $puppetdb_database_host,
+  #   puppetdb_database_replica_host => $puppetdb_database_replica_host,
+  #   compiler_pool_address          => $compiler_pool_address,
+  # )
+
+  if $ha {
+    run_task('pe_xl::configure_node_groups', $master_target,
+      master_host                    => $master_host,
+      master_replica_host            => $master_replica_host,
+      puppetdb_database_host         => $puppetdb_database_host,
+      puppetdb_database_replica_host => $puppetdb_database_replica_host,
+      compiler_pool_address          => $compiler_pool_address,
+    )
+  } else {
+    run_task('pe_xl::configure_node_groups', $master_target,
+      master_host                    => $master_host,
+      puppetdb_database_host         => $puppetdb_database_host,
+      compiler_pool_address          => $compiler_pool_address,
+    )
+  }
 
   # Run Puppet in no-op on the compilers so that their status in PuppetDB
   # is updated and they can be identified by the puppet_enterprise module as
   # CMs
-  run_task('pe_xl::puppet_runonce', [$compiler_hosts, $master_replica_host],
-    noop => true,
-  )
+
+  # SLV-365
+  # run_task('pe_xl::puppet_runonce', [$compiler_hosts, $master_replica_host],
+  #   noop => true,
+  # )
+
+  # in this case assigning a variable seems less efficient...
+  # if $ha {
+  #   $runonce_hosts = [$compiler_hosts, $master_replica_host]
+  # }
+  # else {
+  #   $runonce_hosts = $compiler_hosts
+  # }
+  # 
+  # run_task('pe_xl::puppet_runonce', $runonce_hosts, noop => true)
+
+  if $ha {
+    run_task('pe_xl::puppet_runonce', [$compiler_hosts, $master_replica_host],
+      noop => true,
+    )
+  } else {
+    run_task('pe_xl::puppet_runonce', $compiler_hosts, noop => true)
+  }
 
   # Run Puppet on the PuppetDB Database hosts to update their auth
   # configuration to allow the compilers to connect
-  run_task('pe_xl::puppet_runonce', [
-    $puppetdb_database_host,
-    $puppetdb_database_replica_host,
-  ])
+
+  # SLV-365
+  # run_task('pe_xl::puppet_runonce', [
+  #   $puppetdb_database_host,
+  #   $puppetdb_database_replica_host,
+  # ])
+
+  if $ha {
+    run_task('pe_xl::puppet_runonce', [
+      $puppetdb_database_host,
+      $puppetdb_database_replica_host,
+    ])
+  } else {
+    run_task('pe_xl::puppet_runonce', $puppetdb_database_host)
+  }
 
   # Run Puppet on the master to ensure all services configured and
   # running in prep for provisioning the replica. This is done separately so
@@ -70,24 +122,60 @@
   # other nodes to fail.
   run_task('pe_xl::puppet_runonce', $master_target)
 
+  # SLV-365 
   # Run the PE Replica Provision
-  run_task('pe_xl::provision_replica', $master_target,
-    master_replica         => $master_replica_host,
-    token_file             => $token_file,
-  )
+  # run_task('pe_xl::provision_replica', $master_target,
+  #   master_replica         => $master_replica_host,
+  #   token_file             => $token_file,
+  # )
+
+  # # Run the PE Replica Enable
+  # run_task('pe_xl::enable_replica', $master_target,
+  #   master_replica         => $master_replica_host,
+  #   token_file             => $token_file,
+  # )
+
+  if $ha {
+    # Run the PE Replica Provision
+    run_task('pe_xl::provision_replica', $master_target,
+      master_replica         => $master_replica_host,
+      token_file             => $token_file,
+    )
+
+    # Run the PE Replica Enable
+    run_task('pe_xl::enable_replica', $master_target,
+      master_replica         => $master_replica_host,
+      token_file             => $token_file,
+    )
 
-  # Run the PE Replica Enable
-  run_task('pe_xl::enable_replica', $master_target,
-    master_replica         => $master_replica_host,
-    token_file             => $token_file,
-  )
+  }
 
   # Run Puppet everywhere to pick up last remaining config tweaks
-  run_task('pe_xl::puppet_runonce', [
-    $master_target, $master_replica_host,
-    $puppetdb_database_host, $puppetdb_database_replica_host,
-    $compiler_hosts,
-  ].pe_xl::flatten_compact())
+
+  # SLV-365
+  # run_task('pe_xl::puppet_runonce', [
+  #   $master_target, $master_replica_host,
+  #   $puppetdb_database_host, $puppetdb_database_replica_host,
+  #   $compiler_hosts,
+  # ].pe_xl::flatten_compact())
+
+  if $ha {
+    $all_hosts = [
+      $master_target,
+      $puppetdb_database_host,
+      $compiler_hosts,
+      $master_replica_host,
+      $puppetdb_database_replica_host,
+    ].pe_xl::flatten_compact()
+  } else {
+    $all_hosts = [
+      $master_target,
+      $puppetdb_database_host,
+      $compiler_hosts,
+    ].pe_xl::flatten_compact()
+  }
+
+  run_task('pe_xl::puppet_runonce', $all_hosts)
 
   # Deploy an environment if a deploy environment is specified
   if $deploy_environment {
diff --git a/plans/init.pp b/plans/init.pp
index 56d17698..d9cc10bb 100644
--- a/plans/init.pp
+++ b/plans/init.pp
@@ -8,6 +8,7 @@
   Boolean $install   = false,
   Boolean $configure = false,
   Boolean $upgrade   = false,
+  Boolean $ha        = true,
 
   Optional[String[1]]        $master_host = undef,
   Optional[String[1]]        $puppetdb_database_host = undef,
@@ -29,6 +30,7 @@
 
   if $install {
     run_plan('pe_xl::install',
+      ha                             => $ha,
       master_host                    => $master_host,
       puppetdb_database_host         => $puppetdb_database_host,
       master_replica_host            => $master_replica_host,
@@ -46,6 +48,7 @@
 
   if $configure {
     run_plan('pe_xl::configure',
+      ha                             => $ha,
       master_host                    => $master_host,
       puppetdb_database_host         => $puppetdb_database_host,
       master_replica_host            => $master_replica_host,
@@ -62,6 +65,7 @@
 
   if $upgrade {
     run_plan('pe_xl::upgrade',
+      ha                             => $ha,
       master_host                    => $master_host,
       puppetdb_database_host         => $puppetdb_database_host,
       master_replica_host            => $master_replica_host,
diff --git a/plans/install.pp b/plans/install.pp
index 13725c48..e4fc662a 100644
--- a/plans/install.pp
+++ b/plans/install.pp
@@ -1,6 +1,7 @@
 # @summary Perform initial installation of Puppet Enterprise Extra Large
 #
 plan pe_xl::install (
+  Boolean             $ha,
   String[1]           $master_host,
   String[1]           $puppetdb_database_host,
   String[1]           $master_replica_host,
@@ -15,26 +16,85 @@
   String[1]           $stagingdir = '/tmp',
 ) {
 
+  # TODO: remove 'SLV-365' comments
+
   # Define a number of host groupings for use later in the plan
 
-  $all_hosts = [
+  # SLV-365 - separate core / ha hosts
+  # $all_hosts = [
+  #   $master_host,
+  #   $puppetdb_database_host,
+  #   $compiler_hosts,
+  #   $master_replica_host,
+  #   $puppetdb_database_replica_host,
+  # ].pe_xl::flatten_compact()
+
+  if $ha {
+    out::message('Proceeding with Extra Large HA installation...')
+  } else {
+    out::message('Proceeding with basic Extra Large installation...')
+  }
+
+  $core_hosts = [
     $master_host,
     $puppetdb_database_host,
     $compiler_hosts,
+  ]
+
+  $ha_hosts = [
     $master_replica_host,
     $puppetdb_database_replica_host,
-  ].pe_xl::flatten_compact()
+  ]
+
+  if $ha {
+    $all_hosts = [
+      $core_hosts,
+      $ha_hosts,
+    ].pe_xl::flatten_compact()
+  } else {
+    $all_hosts = [
+      $core_hosts,
+    ].pe_xl::flatten_compact()
+  }
 
-  $pe_installer_hosts = [
-    $master_host,
-    $puppetdb_database_host,
-    $master_replica_host,
-  ].pe_xl::flatten_compact()
+  out::message("all_hosts: ${all_hosts}")
+
+  # SLV-365
+  # $pe_installer_hosts = [
+  #   $master_host,
+  #   $puppetdb_database_host,
+  #   $master_replica_host,
+  # ].pe_xl::flatten_compact()
+
+  if $ha {
+    $pe_installer_hosts = [
+      $master_host,
+      $puppetdb_database_host,
+      $master_replica_host,
+    ].pe_xl::flatten_compact()
+  } else {
+    $pe_installer_hosts = [
+      $master_host,
+      $puppetdb_database_host,
+    ].pe_xl::flatten_compact()
+  }
 
-  $agent_installer_hosts = [
-    $compiler_hosts,
-    $master_replica_host,
-  ].pe_xl::flatten_compact()
+  # SLV-365 -
+  # $agent_installer_hosts = [
+  #   $compiler_hosts,
+  #   $master_replica_host,
+  # ].pe_xl::flatten_compact()
+
+  if $ha {
+    $agent_installer_hosts = [
+      $compiler_hosts,
+      $master_replica_host,
+    ].pe_xl::flatten_compact()
+  } else {
+    $agent_installer_hosts = [
+      $compiler_hosts,
+    ].pe_xl::flatten_compact()
+  }
 
   # There is currently a problem with OID names in csr_attributes.yaml for some
   # installs. Use the raw OIDs for now.
@@ -43,8 +103,15 @@
   $pp_role        = '1.3.6.1.4.1.34380.1.1.13'
 
   # Clusters A and B are used to divide PuppetDB availability for compilers
-  $cm_cluster_a = $compiler_hosts.filter |$index,$cm| { $index % 2 == 0 }
-  $cm_cluster_b = $compiler_hosts.filter |$index,$cm| { $index % 2 != 0 }
+
+  # SLV-365
+  # $cm_cluster_a = $compiler_hosts.filter |$index,$cm| { $index % 2 == 0 }
+  # $cm_cluster_b = $compiler_hosts.filter |$index,$cm| { $index % 2 != 0 }
+
+  if $ha {
+    $cm_cluster_a = $compiler_hosts.filter |$index,$cm| { $index % 2 == 0 }
+    $cm_cluster_b = $compiler_hosts.filter |$index,$cm| { $index % 2 != 0 }
+  }
 
   $dns_alt_names_csv = $dns_alt_names.reduce |$csv,$x| { "${csv},${x}" }
 
@@ -70,23 +137,51 @@
     puppetdb_database_host => $puppetdb_database_host,
   )
 
-  $puppetdb_database_replica_pe_conf = epp('pe_xl/puppetdb_database-pe.conf.epp',
-    master_host            => $master_host,
-    puppetdb_database_host => $puppetdb_database_replica_host,
-  )
+  # SLV-365
+  # $puppetdb_database_replica_pe_conf = epp('pe_xl/puppetdb_database-pe.conf.epp',
+  #   master_host            => $master_host,
+  #   puppetdb_database_host => $puppetdb_database_replica_host,
+  # )
+
+  if $ha {
+    $puppetdb_database_replica_pe_conf = epp('pe_xl/puppetdb_database-pe.conf.epp',
+      master_host            => $master_host,
+      puppetdb_database_host => $puppetdb_database_replica_host,
+    )
+  }
 
   # Upload the pe.conf files to the hosts that need them
   pe_xl::file_content_upload($master_pe_conf, '/tmp/pe.conf', $master_host)
   pe_xl::file_content_upload($puppetdb_database_pe_conf, '/tmp/pe.conf', $puppetdb_database_host)
-  pe_xl::file_content_upload($puppetdb_database_replica_pe_conf, '/tmp/pe.conf', $puppetdb_database_replica_host)
+
+  # SLV-365
+  # pe_xl::file_content_upload($puppetdb_database_replica_pe_conf, '/tmp/pe.conf', $puppetdb_database_replica_host)
+
+  if $ha {
+    pe_xl::file_content_upload($puppetdb_database_replica_pe_conf, '/tmp/pe.conf', $puppetdb_database_replica_host)
+  }
 
   # Download the PE tarball and send it to the nodes that need it
   $pe_tarball_name     = "puppet-enterprise-${version}-el-7-x86_64.tar.gz"
   $local_tarball_path  = "${stagingdir}/${pe_tarball_name}"
   $upload_tarball_path = "/tmp/${pe_tarball_name}"
 
+  # SLV-365
+  # run_plan('pe_xl::util::retrieve_and_upload',
+  #   nodes       => [$master_host, $puppetdb_database_host, $puppetdb_database_replica_host],
+  #   source      => "https://s3.amazonaws.com/pe-builds/released/${version}/puppet-enterprise-${version}-el-7-x86_64.tar.gz",
+  #   local_path  => $local_tarball_path,
+  #   upload_path => $upload_tarball_path,
+  # )
+
+  if $ha {
+    $retrieve_and_upload_hosts = [$master_host, $puppetdb_database_host, $puppetdb_database_replica_host]
+  } else {
+    $retrieve_and_upload_hosts = [$master_host, $puppetdb_database_host]
+  }
+
   run_plan('pe_xl::util::retrieve_and_upload',
-    nodes       => [$master_host, $puppetdb_database_host, $puppetdb_database_replica_host],
+    nodes       => $retrieve_and_upload_hosts,
     source      => "https://s3.amazonaws.com/pe-builds/released/${version}/puppet-enterprise-${version}-el-7-x86_64.tar.gz",
     local_path  => $local_tarball_path,
     upload_path => $upload_tarball_path,
@@ -115,16 +210,30 @@
       | HEREDOC
   )
 
-  run_task('pe_xl::mkdir_p_file', $puppetdb_database_replica_host,
-    path    => '/etc/puppetlabs/puppet/csr_attributes.yaml',
-    content => @("HEREDOC"),
+  # SLV-365
+  # run_task('pe_xl::mkdir_p_file', $puppetdb_database_replica_host,
+  #   path    => '/etc/puppetlabs/puppet/csr_attributes.yaml',
+  #   content => @("HEREDOC"),
+  #     ---
+  #     extension_requests:
+  #       ${pp_application}: "puppet"
+  #       ${pp_role}: "pe_xl::puppetdb_database"
+  #       ${pp_cluster}: "B"
+  #     | HEREDOC
+  # )
+
+  if $ha {
+    run_task('pe_xl::mkdir_p_file', $puppetdb_database_replica_host,
+      path    => '/etc/puppetlabs/puppet/csr_attributes.yaml',
+      content => @("HEREDOC"),
       ---
       extension_requests:
         ${pp_application}: "puppet"
         ${pp_role}: "pe_xl::puppetdb_database"
         ${pp_cluster}: "B"
       | HEREDOC
-  )
+    )
+  }
 
   # Get the master installation up and running. The installer will
   # "fail" because PuppetDB can't start. That's expected.
@@ -140,19 +249,78 @@
   }
 
   # Configure autosigning for the puppetdb database hosts 'cause they need it
-  run_task('pe_xl::mkdir_p_file', $master_host,
-    path    => '/etc/puppetlabs/puppet/autosign.conf',
-    owner   => 'pe-puppet',
-    group   => 'pe-puppet',
-    mode    => '0644',
-    content => @("HEREDOC"),
-      ${puppetdb_database_host}
-      ${puppetdb_database_replica_host}
-      | HEREDOC
-  )
+
+  # SLV-365
+  # run_task('pe_xl::mkdir_p_file', $master_host,
+  #   path    => '/etc/puppetlabs/puppet/autosign.conf',
+  #   owner   => 'pe-puppet',
+  #   group   => 'pe-puppet',
+  #   mode    => '0644',
+  #   content => @("HEREDOC"),
+  #     ${puppetdb_database_host}
+  #     ${puppetdb_database_replica_host}
+  #     | HEREDOC
+  # )
+
+  # TODO: resolve syntax error in this approach
+  # if $ha {
+  #   $content = @("HEREDOC"),
+  #       ${puppetdb_database_host}
+  #       ${puppetdb_database_replica_host}
+  #       | HEREDOC
+  # } else {
+  #   $content = @("HEREDOC"),
+  #       ${puppetdb_database_host}
+  #       | HEREDOC
+  # }
+
+  # run_task('pe_xl::mkdir_p_file', $master_host,
+  #   path    => '/etc/puppetlabs/puppet/autosign.conf',
+  #   owner   => 'pe-puppet',
+  #   group   => 'pe-puppet',
+  #   mode    => '0644',
+  #   content => $content
+  # )
+
+  # TODO: replace with the commented approach above if resolved
+  if $ha {
+    run_task('pe_xl::mkdir_p_file', $master_host,
+      path    => '/etc/puppetlabs/puppet/autosign.conf',
+      owner   => 'pe-puppet',
+      group   => 'pe-puppet',
+      mode    => '0644',
+      content => @("HEREDOC"),
+        ${puppetdb_database_host}
+        ${puppetdb_database_replica_host}
+        | HEREDOC
+    )
+  } else {
+    run_task('pe_xl::mkdir_p_file', $master_host,
+      path    => '/etc/puppetlabs/puppet/autosign.conf',
+      owner   => 'pe-puppet',
+      group   => 'pe-puppet',
+      mode    => '0644',
+      content => @("HEREDOC"),
+        ${puppetdb_database_host}
+        | HEREDOC
+    )
+  }
 
   # Run the PE installer on the puppetdb database hosts
-  run_task('pe_xl::pe_install', [$puppetdb_database_host, $puppetdb_database_replica_host],
+
+  # SLV-365
+  # run_task('pe_xl::pe_install', [$puppetdb_database_host, $puppetdb_database_replica_host],
+  #   tarball => $upload_tarball_path,
+  #   peconf  => '/tmp/pe.conf',
+  # )
+
+  if $ha {
+    $database_hosts = [$puppetdb_database_host, $puppetdb_database_replica_host]
+  } else {
+    $database_hosts = [$puppetdb_database_host]
+  }
+
+  run_task('pe_xl::pe_install', $database_hosts,
     tarball => $upload_tarball_path,
     peconf  => '/tmp/pe.conf',
   )
@@ -184,38 +352,87 @@
   )
 
   # Deploy the PE agent to all remaining hosts
-  run_task('pe_xl::agent_install', $master_replica_host,
-    server        => $master_host,
-    install_flags => [
+  ##############
+  # SLV-365 - clusters A & B?
+  ##############
+  # run_task('pe_xl::agent_install', $master_replica_host,
+  #   server        => $master_host,
+  #   install_flags => [
+  #     '--puppet-service-ensure', 'stopped',
+  #     "main:dns_alt_names=${dns_alt_names_csv}",
+  #     'extension_requests:pp_application=puppet',
+  #     'extension_requests:pp_role=pe_xl::master',
+  #     'extension_requests:pp_cluster=B',
+  #   ],
+  # )
+  #
+  # run_task('pe_xl::agent_install', $cm_cluster_a,
+  #   server        => $master_host,
+  #   install_flags => [
+  #     '--puppet-service-ensure', 'stopped',
+  #     "main:dns_alt_names=${dns_alt_names_csv}",
+  #     'extension_requests:pp_application=puppet',
+  #     'extension_requests:pp_role=pe_xl::compiler',
+  #     'extension_requests:pp_cluster=A',
+  #   ],
+  # )
+  #
+  # run_task('pe_xl::agent_install', $cm_cluster_b,
+  #   server        => $master_host,
+  #   install_flags => [
+  #     '--puppet-service-ensure', 'stopped',
+  #     "main:dns_alt_names=${dns_alt_names_csv}",
+  #     'extension_requests:pp_application=puppet',
+  #     'extension_requests:pp_role=pe_xl::compiler',
+  #     'extension_requests:pp_cluster=B',
+  #   ],
+  # )
+
+  if $ha {
+    run_task('pe_xl::agent_install', $master_replica_host,
+      server        => $master_host,
+      install_flags => [
       '--puppet-service-ensure', 'stopped',
       "main:dns_alt_names=${dns_alt_names_csv}",
       'extension_requests:pp_application=puppet',
       'extension_requests:pp_role=pe_xl::master',
       'extension_requests:pp_cluster=B',
-    ],
-  )
+      ],
+    )
 
-  run_task('pe_xl::agent_install', $cm_cluster_a,
-    server        => $master_host,
-    install_flags => [
+    run_task('pe_xl::agent_install', $cm_cluster_a,
+      server        => $master_host,
+      install_flags => [
       '--puppet-service-ensure', 'stopped',
       "main:dns_alt_names=${dns_alt_names_csv}",
       'extension_requests:pp_application=puppet',
       'extension_requests:pp_role=pe_xl::compiler',
       'extension_requests:pp_cluster=A',
-    ],
-  )
+      ],
+    )
 
-  run_task('pe_xl::agent_install', $cm_cluster_b,
-    server        => $master_host,
-    install_flags => [
+    run_task('pe_xl::agent_install', $cm_cluster_b,
+      server        => $master_host,
+      install_flags => [
       '--puppet-service-ensure', 'stopped',
       "main:dns_alt_names=${dns_alt_names_csv}",
       'extension_requests:pp_application=puppet',
       'extension_requests:pp_role=pe_xl::compiler',
       'extension_requests:pp_cluster=B',
-    ],
-  )
+      ],
+    )
+  } else {
+    run_task('pe_xl::agent_install', $compiler_hosts,
+      server        => $master_host,
+      install_flags => [
+      '--puppet-service-ensure', 'stopped',
+      "main:dns_alt_names=${dns_alt_names_csv}",
+      'extension_requests:pp_application=puppet',
+      'extension_requests:pp_role=pe_xl::compiler',
+      'extension_requests:pp_cluster=A',
+      ],
+    )
+  }
 
   # Do a Puppet agent run to ensure certificate requests have been submitted
   # These runs will "fail", and that's expected.
diff --git a/plans/upgrade.pp b/plans/upgrade.pp
index fad0b218..6e0652ca 100644
--- a/plans/upgrade.pp
+++ b/plans/upgrade.pp
@@ -1,6 +1,7 @@
 # @summary Upgrade an Extra Large stack from one .z to the next
 #
 plan pe_xl::upgrade (
+  Boolean   $ha,
   String[1] $master_host,
   String[1] $puppetdb_database_host,
   String[1] $master_replica_host,
@@ -12,6 +13,8 @@
   String[1] $pe_source  = "https://s3.amazonaws.com/pe-builds/released/${version}/puppet-enterprise-${version}-el-7-x86_64.tar.gz",
 ) {
 
+  # TODO: remove 'SLV-365' comments
+
   # Look up which hosts are compilers in the stack
   # We look up groups of CMs separately since when they are upgraded is determined
   # by which PDB PG host they are affiliated with
@@ -23,22 +26,51 @@
       !(certname = "${master_host}") }
     | PQL
 
-  $compiler_cluster_master_replica_hosts = puppetdb_query(@("PQL")).map |$node| { $node['certname'] }
-    resources[certname] { 
-      type = "Class" and
-      title = "Puppet_enterprise::Profile::Puppetdb" and
-      parameters.database_host = "${puppetdb_database_replica_host}" and
-      !(certname = "${master_replica_host}") }
-    | PQL
+  # SLV-365
+  # $compiler_cluster_master_replica_hosts = puppetdb_query(@("PQL")).map |$node| { $node['certname'] }
+  #   resources[certname] { 
+  #     type = "Class" and
+  #     title = "Puppet_enterprise::Profile::Puppetdb" and
+  #     parameters.database_host = "${puppetdb_database_replica_host}" and
+  #     !(certname = "${master_replica_host}") }
+  #   | PQL
+
+  if $ha {
+    $compiler_cluster_master_replica_hosts = puppetdb_query(@("PQL")).map |$node| { $node['certname'] }
+      resources[certname] { 
+        type = "Class" and
+        title = "Puppet_enterprise::Profile::Puppetdb" and
+        parameters.database_host = "${puppetdb_database_replica_host}" and
+        !(certname = "${master_replica_host}") }
+      | PQL
+  }
 
-  $all_hosts = [
-    $master_host,
-    $puppetdb_database_host,
-    $master_replica_host,
-    $puppetdb_database_replica_host,
-    $compiler_cluster_master_hosts,
-    $compiler_cluster_master_replica_hosts,
-  ].pe_xl::flatten_compact()
+  # SLV-365
+  # $all_hosts = [
+  #   $master_host,
+  #   $puppetdb_database_host,
+  #   $master_replica_host,
+  #   $puppetdb_database_replica_host,
+  #   $compiler_cluster_master_hosts,
+  #   $compiler_cluster_master_replica_hosts,
+  # ].pe_xl::flatten_compact()
+
+  if $ha {
+    $all_hosts = [
+      $master_host,
+      $puppetdb_database_host,
+      $master_replica_host,
+      $puppetdb_database_replica_host,
+      $compiler_cluster_master_hosts,
+      $compiler_cluster_master_replica_hosts,
+    ].pe_xl::flatten_compact()
+  } else {
+    $all_hosts = [
+      $master_host,
+      $puppetdb_database_host,
+      $compiler_cluster_master_hosts,
+    ].pe_xl::flatten_compact()
+  }
 
   $master_local = "local://${master_host}"
 
@@ -47,11 +79,30 @@
   # Download the PE tarball on the nodes that need it
   $upload_tarball_path = "/tmp/puppet-enterprise-${version}-el-7-x86_64.tar.gz"
 
-  run_task('pe_xl::download', [
+  # SLV-365
+  # run_task('pe_xl::download', [
+  #     $master_host,
+  #     $puppetdb_database_host,
+  #     $puppetdb_database_replica_host
+  #   ],
+  #   source => $pe_source,
+  #   path   => $upload_tarball_path,
+  # )
+
+  if $ha {
+    $download_hosts = [
       $master_host,
       $puppetdb_database_host,
-      $puppetdb_database_replica_host
-    ],
+      $puppetdb_database_replica_host,
+    ].pe_xl::flatten_compact()
+  } else {
+    $download_hosts = [
+      $master_host,
+      $puppetdb_database_host,
+    ].pe_xl::flatten_compact()
+  }
+
+  run_task('pe_xl::download', $download_hosts,
     source => $pe_source,
     path   => $upload_tarball_path,
   )
@@ -117,27 +168,52 @@
     server => $master_host,
   )
 
+  # SLV-365
   # Shut down PuppetDB on CMs that use the PMR's PDB PG
-  run_task('service', $compiler_cluster_master_replica_hosts,
-    action => 'stop',
-    name   => 'pe-puppetdb',
-  )
+  # run_task('service', $compiler_cluster_master_replica_hosts,
+  #   action => 'stop',
+  #   name   => 'pe-puppetdb',
+  # )
+
+  # # Run the upgrade.sh script on the master replica host
+  # run_task('pe_xl::agent_upgrade', $master_replica_host,
+  #   server => $master_host,
+  # )
+
+  # # Upgrade the master replica's PuppetDB PostgreSQL host
+  # run_task('pe_xl::pe_install', $puppetdb_database_replica_host,
+  #   tarball => $upload_tarball_path,
+  # )
+  # run_task('pe_xl::puppet_runonce', $puppetdb_database_replica_host)
+
+  # # Upgrade the compiler group B hosts
+  # run_task('pe_xl::agent_upgrade', $compiler_cluster_master_replica_hosts,
+  #   server => $master_host,
+  # )
+
+  if $ha {
+    # Shut down PuppetDB on CMs that use the PMR's PDB PG
+    run_task('service', $compiler_cluster_master_replica_hosts,
+      action => 'stop',
+      name   => 'pe-puppetdb',
+    )
 
-  # Run the upgrade.sh script on the master replica host
-  run_task('pe_xl::agent_upgrade', $master_replica_host,
-    server => $master_host,
-  )
+    # Run the upgrade.sh script on the master replica host
+    run_task('pe_xl::agent_upgrade', $master_replica_host,
+      server => $master_host,
+    )
 
-  # Upgrade the master replica's PuppetDB PostgreSQL host
-  run_task('pe_xl::pe_install', $puppetdb_database_replica_host,
-    tarball => $upload_tarball_path,
-  )
-  run_task('pe_xl::puppet_runonce', $puppetdb_database_replica_host)
+    # Upgrade the master replica's PuppetDB PostgreSQL host
+    run_task('pe_xl::pe_install', $puppetdb_database_replica_host,
+      tarball => $upload_tarball_path,
+    )
+    run_task('pe_xl::puppet_runonce', $puppetdb_database_replica_host)
 
-  # Upgrade the compiler group B hosts
-  run_task('pe_xl::agent_upgrade', $compiler_cluster_master_replica_hosts,
-    server => $master_host,
-  )
+    # Upgrade the compiler group B hosts
+    run_task('pe_xl::agent_upgrade', $compiler_cluster_master_replica_hosts,
+      server => $master_host,
+    )
+  }
 
   # Ensure Puppet running on all infrastructure hosts
   run_task('service', $all_hosts,

From 8fabf7eee7909ee3b10e6dc0a5ec0828cc896b9f Mon Sep 17 00:00:00 2001
From: "bill.claytor" <bill.claytor@puppet.com>
Date: Wed, 10 Jul 2019 12:16:21 -0700
Subject: [PATCH 02/15] (SLV-365) Update pe_xl to remove slv-365 comments

---
 plans/configure.pp |  54 -----------------
 plans/install.pp   | 144 ---------------------------------------------
 plans/upgrade.pp   |  54 -----------------
 3 files changed, 252 deletions(-)

diff --git a/plans/configure.pp b/plans/configure.pp
index eb44e981..d7b62a4a 100644
--- a/plans/configure.pp
+++ b/plans/configure.pp
@@ -45,16 +45,6 @@
 
   # Set up the console node groups to configure the various hosts in their
   # roles
-
-  # SLV-365
-  # run_task('pe_xl::configure_node_groups', $master_target,
-  #   master_host                    => $master_host,
-  #   master_replica_host            => $master_replica_host,
-  #   puppetdb_database_host         => $puppetdb_database_host,
-  #   puppetdb_database_replica_host => $puppetdb_database_replica_host,
-  #   compiler_pool_address          => $compiler_pool_address,
-  # )
-
   if $ha {
     run_task('pe_xl::configure_node_groups', $master_target,
       master_host                    => $master_host,
@@ -74,22 +64,6 @@
   # Run Puppet in no-op on the compilers so that their status in PuppetDB
   # is updated and they can be identified by the puppet_enterprise module as
   # CMs
-
-  # SLV-365
-  # run_task('pe_xl::puppet_runonce', [$compiler_hosts, $master_replica_host],
-  #   noop => true,
-  # )
-
-  # in this case assigning a variable seems less efficient...
-  # if $ha {
-  #   $runonce_hosts = [$compiler_hosts, $master_replica_host]
-  # }
-  # else {
-  #   $runonce_hosts = $compiler_hosts
-  # }
-  # 
-  # run_task('pe_xl::puppet_runonce', $runonce_hosts, noop => true)
-
   if $ha {
     run_task('pe_xl::puppet_runonce', [$compiler_hosts, $master_replica_host],
       noop => true,
@@ -100,13 +74,6 @@
 
   # Run Puppet on the PuppetDB Database hosts to update their auth
   # configuration to allow the compilers to connect
-
-  # SLV-365
-  # run_task('pe_xl::puppet_runonce', [
-  #   $puppetdb_database_host,
-  #   $puppetdb_database_replica_host,
-  # ])
-
   if $ha {
     run_task('pe_xl::puppet_runonce', [
       $puppetdb_database_host,
@@ -122,19 +89,6 @@
   # other nodes to fail.
   run_task('pe_xl::puppet_runonce', $master_target)
 
-  # SLV-365 
-  # Run the PE Replica Provision
-  # run_task('pe_xl::provision_replica', $master_target,
-  #   master_replica         => $master_replica_host,
-  #   token_file             => $token_file,
-  # )
-
-  # # Run the PE Replica Enable
-  # run_task('pe_xl::enable_replica', $master_target,
-  #   master_replica         => $master_replica_host,
-  #   token_file             => $token_file,
-  # )
-
   if $ha {
     # Run the PE Replica Provision
     run_task('pe_xl::provision_replica', $master_target,
@@ -151,14 +105,6 @@
   }
 
   # Run Puppet everywhere to pick up last remaining config tweaks
-
-  # SLV-365
-  # run_task('pe_xl::puppet_runonce', [
-  #   $master_target, $master_replica_host,
-  #   $puppetdb_database_host, $puppetdb_database_replica_host,
-  #   $compiler_hosts,
-  # ].pe_xl::flatten_compact())
-
   if $ha {
     $all_hosts = [
       $master_target,
diff --git a/plans/install.pp b/plans/install.pp
index e4fc662a..9af2dba7 100644
--- a/plans/install.pp
+++ b/plans/install.pp
@@ -16,25 +16,7 @@
   String[1]           $stagingdir = '/tmp',
 ) {
 
-  # TODO: remove 'SLV-365' comments
-
   # Define a number of host groupings for use later in the plan
-
-  # SLV-365 - separate core / ha hosts
-  # $all_hosts = [
-  #   $master_host,
-  #   $puppetdb_database_host,
-  #   $compiler_hosts,
-  #   $master_replica_host,
-  #   $puppetdb_database_replica_host,
-  # ].pe_xl::flatten_compact()
-
-  if $ha {
-    out::message('Proceeding with Extra Large HA installation...')
-  } else {
-    out::message('Proceeding with basic Extra Large installation...')
-  }
-
   $core_hosts = [
     $master_host,
     $puppetdb_database_host,
@@ -57,15 +39,6 @@
     ].pe_xl::flatten_compact()
   }
 
-  out::message("all_hosts: ${all_hosts}")
-
-  # SLV-365
-  # $pe_installer_hosts = [
-  #   $master_host,
-  #   $puppetdb_database_host,
-  #   $master_replica_host,
-  # ].pe_xl::flatten_compact()
-
   if $ha {
     $pe_installer_hosts = [
       $master_host,
@@ -79,12 +52,6 @@
     ].pe_xl::flatten_compact()
   }
 
-  # SLV-365 -
-  # $agent_installer_hosts = [
-  #   $compiler_hosts,
-  #   $master_replica_host,
-  # ].pe_xl::flatten_compact()
-
   if $ha {
     $agent_installer_hosts = [
       $compiler_hosts,
@@ -103,11 +70,6 @@
   $pp_role        = '1.3.6.1.4.1.34380.1.1.13'
 
   # Clusters A and B are used to divide PuppetDB availability for compilers
-
-  # SLV-365
-  # $cm_cluster_a = $compiler_hosts.filter |$index,$cm| { $index % 2 == 0 }
-  # $cm_cluster_b = $compiler_hosts.filter |$index,$cm| { $index % 2 != 0 }
-
   if $ha {
     $cm_cluster_a = $compiler_hosts.filter |$index,$cm| { $index % 2 == 0 }
     $cm_cluster_b = $compiler_hosts.filter |$index,$cm| { $index % 2 != 0 }
@@ -137,12 +99,6 @@
     puppetdb_database_host => $puppetdb_database_host,
   )
 
-  # SLV-365
-  # $puppetdb_database_replica_pe_conf = epp('pe_xl/puppetdb_database-pe.conf.epp',
-  #   master_host            => $master_host,
-  #   puppetdb_database_host => $puppetdb_database_replica_host,
-  # )
-
   if $ha {
     $puppetdb_database_replica_pe_conf = epp('pe_xl/puppetdb_database-pe.conf.epp',
       master_host            => $master_host,
@@ -154,9 +110,6 @@
   pe_xl::file_content_upload($master_pe_conf, '/tmp/pe.conf', $master_host)
   pe_xl::file_content_upload($puppetdb_database_pe_conf, '/tmp/pe.conf', $puppetdb_database_host)
 
-  # SLV-365
-  # pe_xl::file_content_upload($puppetdb_database_replica_pe_conf, '/tmp/pe.conf', $puppetdb_database_replica_host)
-
   if $ha {
     pe_xl::file_content_upload($puppetdb_database_replica_pe_conf, '/tmp/pe.conf', $puppetdb_database_replica_host)
   }
@@ -166,14 +119,6 @@
   $local_tarball_path  = "${stagingdir}/${pe_tarball_name}"
   $upload_tarball_path = "/tmp/${pe_tarball_name}"
 
-  # SLV-365
-  # run_plan('pe_xl::util::retrieve_and_upload',
-  #   nodes       => [$master_host, $puppetdb_database_host, $puppetdb_database_replica_host],
-  #   source      => "https://s3.amazonaws.com/pe-builds/released/${version}/puppet-enterprise-${version}-el-7-x86_64.tar.gz",
-  #   local_path  => $local_tarball_path,
-  #   upload_path => $upload_tarball_path,
-  # )
-
   if $ha {
     $retrieve_and_upload_hosts = [$master_host, $puppetdb_database_host, $puppetdb_database_replica_host]
   } else {
@@ -210,18 +155,6 @@
       | HEREDOC
   )
 
-  # SLV-365
-  # run_task('pe_xl::mkdir_p_file', $puppetdb_database_replica_host,
-  #   path    => '/etc/puppetlabs/puppet/csr_attributes.yaml',
-  #   content => @("HEREDOC"),
-  #     ---
-  #     extension_requests:
-  #       ${pp_application}: "puppet"
-  #       ${pp_role}: "pe_xl::puppetdb_database"
-  #       ${pp_cluster}: "B"
-  #     | HEREDOC
-  # )
-
   if $ha {
     run_task('pe_xl::mkdir_p_file', $puppetdb_database_replica_host,
       path    => '/etc/puppetlabs/puppet/csr_attributes.yaml',
@@ -249,40 +182,6 @@
   }
 
   # Configure autosigning for the puppetdb database hosts 'cause they need it
-
-  # SLV-365
-  # run_task('pe_xl::mkdir_p_file', $master_host,
-  #   path    => '/etc/puppetlabs/puppet/autosign.conf',
-  #   owner   => 'pe-puppet',
-  #   group   => 'pe-puppet',
-  #   mode    => '0644',
-  #   content => @("HEREDOC"),
-  #     ${puppetdb_database_host}
-  #     ${puppetdb_database_replica_host}
-  #     | HEREDOC
-  # )
-
-  # TODO: resolve syntax error in this approach
-  # if $ha {
-  #   $content = @("HEREDOC"),
-  #       ${puppetdb_database_host}
-  #       ${puppetdb_database_replica_host}
-  #       | HEREDOC
-  # } else {
-  #   $content = @("HEREDOC"),
-  #       ${puppetdb_database_host}
-  #       | HEREDOC
-  # }
-
-  # run_task('pe_xl::mkdir_p_file', $master_host,
-  #   path    => '/etc/puppetlabs/puppet/autosign.conf',
-  #   owner   => 'pe-puppet',
-  #   group   => 'pe-puppet',
-  #   mode    => '0644',
-  #   content => $content
-  # )
-
-  # TODO: replace with the commented approach above if resolved
   if $ha {
     run_task('pe_xl::mkdir_p_file', $master_host,
       path    => '/etc/puppetlabs/puppet/autosign.conf',
@@ -307,13 +206,6 @@
   }
 
   # Run the PE installer on the puppetdb database hosts
-
-  # SLV-365
-  # run_task('pe_xl::pe_install', [$puppetdb_database_host, $puppetdb_database_replica_host],
-  #   tarball => $upload_tarball_path,
-  #   peconf  => '/tmp/pe.conf',
-  # )
-
   if $ha {
     $database_hosts = [$puppetdb_database_host, $puppetdb_database_replica_host]
   } else {
@@ -352,42 +244,6 @@
   )
 
   # Deploy the PE agent to all remaining hosts
-  ##############
-  # SLV-365 - clusters A & B?
-  ##############
-  # run_task('pe_xl::agent_install', $master_replica_host,
-  #   server        => $master_host,
-  #   install_flags => [
-  #     '--puppet-service-ensure', 'stopped',
-  #     "main:dns_alt_names=${dns_alt_names_csv}",
-  #     'extension_requests:pp_application=puppet',
-  #     'extension_requests:pp_role=pe_xl::master',
-  #     'extension_requests:pp_cluster=B',
-  #   ],
-  # )
-  #
-  # run_task('pe_xl::agent_install', $cm_cluster_a,
-  #   server        => $master_host,
-  #   install_flags => [
-  #     '--puppet-service-ensure', 'stopped',
-  #     "main:dns_alt_names=${dns_alt_names_csv}",
-  #     'extension_requests:pp_application=puppet',
-  #     'extension_requests:pp_role=pe_xl::compiler',
-  #     'extension_requests:pp_cluster=A',
-  #   ],
-  # )
-  #
-  # run_task('pe_xl::agent_install', $cm_cluster_b,
-  #   server        => $master_host,
-  #   install_flags => [
-  #     '--puppet-service-ensure', 'stopped',
-  #     "main:dns_alt_names=${dns_alt_names_csv}",
-  #     'extension_requests:pp_application=puppet',
-  #     'extension_requests:pp_role=pe_xl::compiler',
-  #     'extension_requests:pp_cluster=B',
-  #   ],
-  # )
-
   if $ha {
     run_task('pe_xl::agent_install', $master_replica_host,
       server        => $master_host,
diff --git a/plans/upgrade.pp b/plans/upgrade.pp
index 6e0652ca..a2513615 100644
--- a/plans/upgrade.pp
+++ b/plans/upgrade.pp
@@ -13,8 +13,6 @@
   String[1] $pe_source  = "https://s3.amazonaws.com/pe-builds/released/${version}/puppet-enterprise-${version}-el-7-x86_64.tar.gz",
 ) {
 
-  # TODO: remove 'SLV-365' comments
-
   # Look up which hosts are compilers in the stack
   # We look up groups of CMs separately since when they are upgraded is determined
   # by which PDB PG host they are affiliated with
@@ -26,15 +24,6 @@
       !(certname = "${master_host}") }
     | PQL
 
-  # SLV-365
-  # $compiler_cluster_master_replica_hosts = puppetdb_query(@("PQL")).map |$node| { $node['certname'] }
-  #   resources[certname] { 
-  #     type = "Class" and
-  #     title = "Puppet_enterprise::Profile::Puppetdb" and
-  #     parameters.database_host = "${puppetdb_database_replica_host}" and
-  #     !(certname = "${master_replica_host}") }
-  #   | PQL
-
   if $ha {
     $compiler_cluster_master_replica_hosts = puppetdb_query(@("PQL")).map |$node| { $node['certname'] }
       resources[certname] { 
@@ -45,16 +34,6 @@
       | PQL
   }
 
-  # SLV-365
-  # $all_hosts = [
-  #   $master_host,
-  #   $puppetdb_database_host,
-  #   $master_replica_host,
-  #   $puppetdb_database_replica_host,
-  #   $compiler_cluster_master_hosts,
-  #   $compiler_cluster_master_replica_hosts,
-  # ].pe_xl::flatten_compact()
-
   if $ha {
     $all_hosts = [
       $master_host,
@@ -79,16 +58,6 @@
   # Download the PE tarball on the nodes that need it
   $upload_tarball_path = "/tmp/puppet-enterprise-${version}-el-7-x86_64.tar.gz"
 
-  # SLV-365
-  # run_task('pe_xl::download', [
-  #     $master_host,
-  #     $puppetdb_database_host,
-  #     $puppetdb_database_replica_host
-  #   ],
-  #   source => $pe_source,
-  #   path   => $upload_tarball_path,
-  # )
-
   if $ha {
     $download_hosts = [
       $master_host,
@@ -168,29 +137,6 @@
     server => $master_host,
   )
 
-  # SLV-365
-  # Shut down PuppetDB on CMs that use the PMR's PDB PG
-  # run_task('service', $compiler_cluster_master_replica_hosts,
-  #   action => 'stop',
-  #   name   => 'pe-puppetdb',
-  # )
-
-  # # Run the upgrade.sh script on the master replica host
-  # run_task('pe_xl::agent_upgrade', $master_replica_host,
-  #   server => $master_host,
-  # )
-
-  # # Upgrade the master replica's PuppetDB PostgreSQL host
-  # run_task('pe_xl::pe_install', $puppetdb_database_replica_host,
-  #   tarball => $upload_tarball_path,
-  # )
-  # run_task('pe_xl::puppet_runonce', $puppetdb_database_replica_host)
-
-  # # Upgrade the compiler group B hosts
-  # run_task('pe_xl::agent_upgrade', $compiler_cluster_master_replica_hosts,
-  #   server => $master_host,
-  # )
-
   if $ha {
     # Shut down PuppetDB on CMs that use the PMR's PDB PG
     run_task('service', $compiler_cluster_master_replica_hosts,

From 5ff4542f8eebae71ef8d2758414297378c1ccaaa Mon Sep 17 00:00:00 2001
From: "bill.claytor" <bill.claytor@puppet.com>
Date: Wed, 10 Jul 2019 12:29:21 -0700
Subject: [PATCH 03/15] (SLV-365) Removed comment

---
 plans/configure.pp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/plans/configure.pp b/plans/configure.pp
index d7b62a4a..66e11a2b 100644
--- a/plans/configure.pp
+++ b/plans/configure.pp
@@ -22,8 +22,6 @@
   String[1]           $stagingdir = '/tmp',
 ) {
 
-  # TODO: remove 'SLV-365' comments
-
   # Allow for the configure task to be run local to the master.
   $master_target = $executing_on_master ? {
     true  => "local://${master_host}",

From dbe46f6f60e5cbbc7cb5d99c2a06def2dbb5912b Mon Sep 17 00:00:00 2001
From: "bill.claytor" <bill.claytor@puppet.com>
Date: Fri, 12 Jul 2019 06:35:14 -0700
Subject: [PATCH 04/15] (SLV-365) Updated plans to set ha = true by default

---
 plans/configure.pp | 2 +-
 plans/install.pp   | 2 +-
 plans/upgrade.pp   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/plans/configure.pp b/plans/configure.pp
index 66e11a2b..ce3f1857 100644
--- a/plans/configure.pp
+++ b/plans/configure.pp
@@ -1,7 +1,7 @@
 # @summary Configure first-time classification and HA setup
 #
 plan pe_xl::configure (
-  Boolean             $ha,
+  Boolean             $ha = true,
   String[1]           $master_host,
   String[1]           $puppetdb_database_host,
   String[1]           $master_replica_host,
diff --git a/plans/install.pp b/plans/install.pp
index 9af2dba7..30abfc80 100644
--- a/plans/install.pp
+++ b/plans/install.pp
@@ -1,7 +1,7 @@
 # @summary Perform initial installation of Puppet Enterprise Extra Large
 #
 plan pe_xl::install (
-  Boolean             $ha,
+  Boolean             $ha = true,
   String[1]           $master_host,
   String[1]           $puppetdb_database_host,
   String[1]           $master_replica_host,
diff --git a/plans/upgrade.pp b/plans/upgrade.pp
index a2513615..a3df3905 100644
--- a/plans/upgrade.pp
+++ b/plans/upgrade.pp
@@ -1,7 +1,7 @@
 # @summary Upgrade an Extra Large stack from one .z to the next
 #
 plan pe_xl::upgrade (
-  Boolean   $ha,
+  Boolean   $ha = true,
   String[1] $master_host,
   String[1] $puppetdb_database_host,
   String[1] $master_replica_host,

From fef2ea50b17d0a4dd140d27a356816865591c321 Mon Sep 17 00:00:00 2001
From: "bill.claytor" <bill.claytor@puppet.com>
Date: Fri, 12 Jul 2019 11:58:13 -0700
Subject: [PATCH 05/15] (SLV-365) Added install_and_configure_without_ha.md

---
 .../install_and_configure_without_ha.md       | 67 +++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 documentation/install_and_configure_without_ha.md

diff --git a/documentation/install_and_configure_without_ha.md b/documentation/install_and_configure_without_ha.md
new file mode 100644
index 00000000..18349037
--- /dev/null
+++ b/documentation/install_and_configure_without_ha.md
@@ -0,0 +1,67 @@
+# Install and configure Extra Large without HA
+
+* TODO: add this doc as a section to basic_usage.md instead?
+
+Please see the [basic_usage.md](basic_usage.md) document for reference; this document will avoid repeating the information covered there.
+The plans covered in this document can also set up the Extra Large environment without HA by setting the optional `ha` parameter to `false` in the params.json file.
+
+## Basic usage instructions
+
+1. Ensure the hostname of each system is set correctly, to the same value that will be used to connect to the system, and refer to the system as. If the hostname is not set as expected the installation plan will refuse to continue.
+2. Install Bolt on a jumphost. This can be the master, or any other system.
+3. Download or git clone the pe\_xl module and put it somewhere on the jumphost, e.g. ~/modules/pe\_xl.
+4. Create an inventory file with connection information. An example is included below.
+5. Create a parameters file. An example is included below. Note the addition of the "ha" parameter with a value of "false", and the omission of the "replica" roles.
+6. Run the pe\_xl plan with the inputs created. Example:
+```
+        bolt plan run pe_xl \
+          --inventory nodes.yaml \
+          --modulepath ~/modules \
+          --params @params.json 
+```
+
+Example nodes.yaml Bolt inventory file:
+
+```yaml
+
+---
+groups:
+  - name: pe_xl_nodes
+    config:
+      transport: ssh
+      ssh:
+        host-key-check: false
+        user: centos
+        run-as: root
+        tty: true
+    nodes:
+      - pe-xl-core-0.lab1.puppet.vm
+      - pe-xl-core-1.lab1.puppet.vm
+      - pe-xl-compiler-0.lab1.puppet.vm
+      - pe-xl-compiler-1.lab1.puppet.vm
+```
+
+Example params.json Bolt parameters file:
+
+```json
+{
+  "install": true,
+  "configure": true,
+  "upgrade": false,
+  "ha": false,
+
+  "master_host": "pe-xl-core-0.lab1.puppet.vm",
+  "puppetdb_database_host": "pe-xl-core-1.lab1.puppet.vm",
+  "master_replica_host": "",
+  "puppetdb_database_replica_host": "",
+  "compiler_hosts": [
+    "pe-xl-compiler-0.lab1.puppet.vm",
+    "pe-xl-compiler-1.lab1.puppet.vm"
+  ],
+
+  "console_password": "puppetlabs",
+  "dns_alt_names": [ "puppet", "puppet.lab1.puppet.vm" ],
+  "compiler_pool_address": "puppet.lab1.puppet.vm",
+  "version": "2019.1.0"
+}
+```

From a1da0b8aa905538733cd82beea7ca2876afeba78 Mon Sep 17 00:00:00 2001
From: "bill.claytor" <bill.claytor@puppet.com>
Date: Fri, 12 Jul 2019 12:09:33 -0700
Subject: [PATCH 06/15] (SLV-365) Minor fixes to
 install_and_configure_without_ha.md

---
 documentation/install_and_configure_without_ha.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/documentation/install_and_configure_without_ha.md b/documentation/install_and_configure_without_ha.md
index 18349037..ea0df2f7 100644
--- a/documentation/install_and_configure_without_ha.md
+++ b/documentation/install_and_configure_without_ha.md
@@ -3,7 +3,7 @@
 * TODO: add this doc as a section to basic_usage.md instead?
 
 Please see the [basic_usage.md](basic_usage.md) document for reference; this document will avoid repeating the information covered there.
-The plans covered in this document can also set up the Extra Large environment without HA by setting the optional `ha` parameter to `false` in the params.json file.
+The install, configure, and upgrade plans covered in the [basic_usage.md](basic_usage.md) document can also set up the Extra Large environment without HA by setting the optional `ha` parameter to `false` in the params.json file (see the [example](#example-params.json-bolt-parameters-file) below).
 
 ## Basic usage instructions
 
@@ -20,7 +20,7 @@ The plans covered in this document can also set up the Extra Large environment w
           --params @params.json 
 ```
 
-Example nodes.yaml Bolt inventory file:
+### Example nodes.yaml Bolt inventory file
 
 ```yaml
 
@@ -41,7 +41,7 @@ groups:
       - pe-xl-compiler-1.lab1.puppet.vm
 ```
 
-Example params.json Bolt parameters file:
+### Example params.json Bolt parameters file
 
 ```json
 {

From 33a300c46de65465cbca90a2cfa058b4339f7f15 Mon Sep 17 00:00:00 2001
From: "bill.claytor" <bill.claytor@puppet.com>
Date: Wed, 17 Jul 2019 12:19:58 -0700
Subject: [PATCH 07/15] (SLV-365) Fixed formatting issues in install.pp

---
 plans/install.pp | 52 ++++++++++++++++++++++++------------------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/plans/install.pp b/plans/install.pp
index 30abfc80..56bc5dba 100644
--- a/plans/install.pp
+++ b/plans/install.pp
@@ -159,12 +159,12 @@
     run_task('pe_xl::mkdir_p_file', $puppetdb_database_replica_host,
       path    => '/etc/puppetlabs/puppet/csr_attributes.yaml',
       content => @("HEREDOC"),
-      ---
-      extension_requests:
-        ${pp_application}: "puppet"
-        ${pp_role}: "pe_xl::puppetdb_database"
-        ${pp_cluster}: "B"
-      | HEREDOC
+        ---
+        extension_requests:
+          ${pp_application}: "puppet"
+          ${pp_role}: "pe_xl::puppetdb_database"
+          ${pp_cluster}: "B"
+        | HEREDOC
     )
   }
 
@@ -248,44 +248,44 @@
     run_task('pe_xl::agent_install', $master_replica_host,
       server        => $master_host,
       install_flags => [
-      '--puppet-service-ensure', 'stopped',
-      "main:dns_alt_names=${dns_alt_names_csv}",
-      'extension_requests:pp_application=puppet',
-      'extension_requests:pp_role=pe_xl::master',
-      'extension_requests:pp_cluster=B',
+        '--puppet-service-ensure', 'stopped',
+        "main:dns_alt_names=${dns_alt_names_csv}",
+        'extension_requests:pp_application=puppet',
+        'extension_requests:pp_role=pe_xl::master',
+        'extension_requests:pp_cluster=B',
       ],
     )
 
     run_task('pe_xl::agent_install', $cm_cluster_a,
       server        => $master_host,
       install_flags => [
-      '--puppet-service-ensure', 'stopped',
-      "main:dns_alt_names=${dns_alt_names_csv}",
-      'extension_requests:pp_application=puppet',
-      'extension_requests:pp_role=pe_xl::compiler',
-      'extension_requests:pp_cluster=A',
+        '--puppet-service-ensure', 'stopped',
+        "main:dns_alt_names=${dns_alt_names_csv}",
+        'extension_requests:pp_application=puppet',
+        'extension_requests:pp_role=pe_xl::compiler',
+        'extension_requests:pp_cluster=A',
       ],
     )
 
     run_task('pe_xl::agent_install', $cm_cluster_b,
       server        => $master_host,
       install_flags => [
-      '--puppet-service-ensure', 'stopped',
-      "main:dns_alt_names=${dns_alt_names_csv}",
-      'extension_requests:pp_application=puppet',
-      'extension_requests:pp_role=pe_xl::compiler',
-      'extension_requests:pp_cluster=B',
+        '--puppet-service-ensure', 'stopped',
+        "main:dns_alt_names=${dns_alt_names_csv}",
+        'extension_requests:pp_application=puppet',
+        'extension_requests:pp_role=pe_xl::compiler',
+        'extension_requests:pp_cluster=B',
       ],
     )
   } else {
     run_task('pe_xl::agent_install', $compiler_hosts,
       server        => $master_host,
       install_flags => [
-      '--puppet-service-ensure', 'stopped',
-      "main:dns_alt_names=${dns_alt_names_csv}",
-      'extension_requests:pp_application=puppet',
-      'extension_requests:pp_role=pe_xl::compiler',
-      'extension_requests:pp_cluster=A',
+        '--puppet-service-ensure', 'stopped',
+        "main:dns_alt_names=${dns_alt_names_csv}",
+        'extension_requests:pp_application=puppet',
+        'extension_requests:pp_role=pe_xl::compiler',
+        'extension_requests:pp_cluster=A',
       ],
     )
   }

From 55bb7c845389de2d86fce506fc2208f3d0bbc120 Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppetlabs.com>
Date: Tue, 30 Jul 2019 13:48:23 -0700
Subject: [PATCH 08/15] Streamline implementation of optional HA

Try to reduce the number of `if` statements in code, and remove the need
to specify `ha` as a Boolean plan parameter. Instead, do HA if HA
systems are specified, and don't if they aren't.
---
 plans/configure.pp                      |  87 ++++-----
 plans/init.pp                           |   4 -
 plans/install.pp                        | 232 ++++++++++--------------
 plans/upgrade.pp                        | 108 +++++------
 templates/puppetdb_database-pe.conf.epp |   2 +-
 5 files changed, 180 insertions(+), 253 deletions(-)

diff --git a/plans/configure.pp b/plans/configure.pp
index ce3f1857..8d544564 100644
--- a/plans/configure.pp
+++ b/plans/configure.pp
@@ -1,13 +1,13 @@
 # @summary Configure first-time classification and HA setup
 #
 plan pe_xl::configure (
-  Boolean             $ha = true,
   String[1]           $master_host,
   String[1]           $puppetdb_database_host,
-  String[1]           $master_replica_host,
-  String[1]           $puppetdb_database_replica_host,
   Array[String[1]]    $compiler_hosts = [ ],
 
+  Optional[String[1]] $master_replica_host = undef,
+  Optional[String[1]] $puppetdb_database_replica_host = undef,
+
   # This parameter exists primarily to enable the use case of running
   # pe_xl::configure over the PCP transport. An orchestrator restart happens
   # during provision replica. Running `bolt plan run` directly on the master
@@ -22,6 +22,18 @@
   String[1]           $stagingdir = '/tmp',
 ) {
 
+  $ha_hosts = [
+    $master_replica_host,
+    $puppetdb_database_replica_host,
+  ].pe_xl::flatten_compact()
+
+  # Ensure valid input for HA
+  $ha = $ha_hosts.size ? {
+    0       => false,
+    2       => true,
+    default => fail("Must specify either both or neither of master_replica_host, puppetdb_database_replica_host"),
+  }
+
   # Allow for the configure task to be run local to the master.
   $master_target = $executing_on_master ? {
     true  => "local://${master_host}",
@@ -43,43 +55,27 @@
 
   # Set up the console node groups to configure the various hosts in their
   # roles
-  if $ha {
-    run_task('pe_xl::configure_node_groups', $master_target,
-      master_host                    => $master_host,
-      master_replica_host            => $master_replica_host,
-      puppetdb_database_host         => $puppetdb_database_host,
-      puppetdb_database_replica_host => $puppetdb_database_replica_host,
-      compiler_pool_address          => $compiler_pool_address,
-    )
-  } else {
-    run_task('pe_xl::configure_node_groups', $master_target,
-      master_host                    => $master_host,
-      puppetdb_database_host         => $puppetdb_database_host,
-      compiler_pool_address          => $compiler_pool_address,
-    )
-  }
+  run_task('pe_xl::configure_node_groups', $master_target,
+    master_host                    => $master_host,
+    master_replica_host            => $master_replica_host,
+    puppetdb_database_host         => $puppetdb_database_host,
+    puppetdb_database_replica_host => $puppetdb_database_replica_host,
+    compiler_pool_address          => $compiler_pool_address,
+  )
 
   # Run Puppet in no-op on the compilers so that their status in PuppetDB
   # is updated and they can be identified by the puppet_enterprise module as
   # CMs
-  if $ha {
-    run_task('pe_xl::puppet_runonce', [$compiler_hosts, $master_replica_host],
-      noop => true,
-    )
-  } else {
-    run_task('pe_xl::puppet_runonce', $compiler_hosts, noop => true)
-  }
+  run_task('pe_xl::puppet_runonce', [$compiler_hosts, $master_replica_host].pe_xl::flatten_compact,
+    noop => true,
+  )
 
   # Run Puppet on the PuppetDB Database hosts to update their auth
   # configuration to allow the compilers to connect
-  if $ha {
-    run_task('pe_xl::puppet_runonce', [
-      $puppetdb_database_host,
-      $puppetdb_database_replica_host,
-    ])
-  } else {
-    run_task('pe_xl::puppet_runonce', $puppetdb_database_host)
-  }
+  run_task('pe_xl::puppet_runonce', [
+    $puppetdb_database_host,
+    $puppetdb_database_replica_host,
+  ].pe_xl::flatten_compact)
 
   # Run Puppet on the master to ensure all services configured and
   # running in prep for provisioning the replica. This is done separately so
@@ -99,27 +95,16 @@
       master_replica         => $master_replica_host,
       token_file             => $token_file,
     )
-
   }
 
   # Run Puppet everywhere to pick up last remaining config tweaks
-  if $ha {
-    $all_hosts = [
-      $master_target,
-      $puppetdb_database_host,
-      $compiler_hosts,
-      $master_replica_host,
-      $puppetdb_database_replica_host,
-    ].pe_xl::flatten_compact()
-  } else {
-    $all_hosts = [
-      $master_target,
-      $puppetdb_database_host,
-      $compiler_hosts,
-    ].pe_xl::flatten_compact()
-  }
-
-  run_task('pe_xl::puppet_runonce', $all_hosts)
+  run_task('pe_xl::puppet_runonce', [
+    $master_target,
+    $puppetdb_database_host,
+    $compiler_hosts,
+    $master_replica_host,
+    $puppetdb_database_replica_host,
+  ].pe_xl::flatten_compact)
 
   # Deploy an environment if a deploy environment is specified
   if $deploy_environment {
diff --git a/plans/init.pp b/plans/init.pp
index d9cc10bb..56d17698 100644
--- a/plans/init.pp
+++ b/plans/init.pp
@@ -8,7 +8,6 @@
   Boolean $install   = false,
   Boolean $configure = false,
   Boolean $upgrade   = false,
-  Boolean $ha        = true,
 
   Optional[String[1]]        $master_host = undef,
   Optional[String[1]]        $puppetdb_database_host = undef,
@@ -30,7 +29,6 @@
 
   if $install {
     run_plan('pe_xl::install',
-      ha                             => $ha,
       master_host                    => $master_host,
       puppetdb_database_host         => $puppetdb_database_host,
       master_replica_host            => $master_replica_host,
@@ -48,7 +46,6 @@
 
   if $configure {
     run_plan('pe_xl::configure',
-      ha                             => $ha,
       master_host                    => $master_host,
       puppetdb_database_host         => $puppetdb_database_host,
       master_replica_host            => $master_replica_host,
@@ -65,7 +62,6 @@
 
   if $upgrade {
     run_plan('pe_xl::upgrade',
-      ha                             => $ha,
       master_host                    => $master_host,
       puppetdb_database_host         => $puppetdb_database_host,
       master_replica_host            => $master_replica_host,
diff --git a/plans/install.pp b/plans/install.pp
index 56bc5dba..31826fdb 100644
--- a/plans/install.pp
+++ b/plans/install.pp
@@ -1,13 +1,13 @@
 # @summary Perform initial installation of Puppet Enterprise Extra Large
 #
 plan pe_xl::install (
-  Boolean             $ha = true,
   String[1]           $master_host,
   String[1]           $puppetdb_database_host,
-  String[1]           $master_replica_host,
-  String[1]           $puppetdb_database_replica_host,
   Array[String[1]]    $compiler_hosts = [ ],
 
+  Optional[String[1]] $master_replica_host = undef,
+  Optional[String[1]] $puppetdb_database_replica_host = undef,
+
   String[1]           $console_password,
   String[1]           $version = '2018.1.3',
   Hash                $r10k_sources = { },
@@ -20,49 +20,50 @@
   $core_hosts = [
     $master_host,
     $puppetdb_database_host,
-    $compiler_hosts,
-  ]
+  ].pe_xl::flatten_compact()
 
   $ha_hosts = [
     $master_replica_host,
     $puppetdb_database_replica_host,
-  ]
+  ].pe_xl::flatten_compact()
 
-  if $ha {
-    $all_hosts = [
-      $core_hosts,
-      $ha_hosts,
-    ].pe_xl::flatten_compact()
-  } else {
-    $all_hosts = [
-      $core_hosts,
-    ].pe_xl::flatten_compact()
-  }
+  $ha_replica_target = [
+    $master_replica_host,
+  ].pe_xl::flatten_compact()
 
-  if $ha {
-    $pe_installer_hosts = [
-      $master_host,
-      $puppetdb_database_host,
-      $master_replica_host,
-    ].pe_xl::flatten_compact()
-  } else {
-    $pe_installer_hosts = [
-      $master_host,
-      $puppetdb_database_host,
-    ].pe_xl::flatten_compact()
-  }
+  $ha_database_target = [
+    $puppetdb_database_replica_host,
+  ].pe_xl::flatten_compact()
 
-  if $ha {
-    $agent_installer_hosts = [
-      $compiler_hosts,
-      $master_replica_host,
-    ].pe_xl::flatten_compact()
-  } else {
-    $agent_installer_hosts = [
-      $compiler_hosts,
-    ].pe_xl::flatten_compact()
+  # Ensure valid input for HA
+  $ha = $ha_hosts.size ? {
+    0       => false,
+    2       => true,
+    default => fail("Must specify either both or neither of master_replica_host, puppetdb_database_replica_host"),
   }
 
+  $all_hosts = [
+    $core_hosts,
+    $ha_hosts,
+    $compiler_hosts,
+  ].pe_xl::flatten_compact()
+
+  $database_hosts = [
+    $puppetdb_database_host,
+    $puppetdb_database_replica_host,
+  ].pe_xl::flatten_compact()
+
+  $pe_installer_hosts = [
+    $master_host,
+    $puppetdb_database_host,
+    $master_replica_host,
+  ].pe_xl::flatten_compact()
+
+  $agent_installer_hosts = [
+    $compiler_hosts,
+    $master_replica_host,
+  ].pe_xl::flatten_compact()
+
   # There is currently a problem with OID names in csr_attributes.yaml for some
   # installs. Use the raw OIDs for now.
   $pp_application = '1.3.6.1.4.1.34380.1.1.8'
@@ -74,6 +75,10 @@
     $cm_cluster_a = $compiler_hosts.filter |$index,$cm| { $index % 2 == 0 }
     $cm_cluster_b = $compiler_hosts.filter |$index,$cm| { $index % 2 != 0 }
   }
+  else {
+    $cm_cluster_a = $compiler_hosts
+    $cm_cluster_b = []
+  }
 
   $dns_alt_names_csv = $dns_alt_names.reduce |$csv,$x| { "${csv},${x}" }
 
@@ -99,34 +104,23 @@
     puppetdb_database_host => $puppetdb_database_host,
   )
 
-  if $ha {
-    $puppetdb_database_replica_pe_conf = epp('pe_xl/puppetdb_database-pe.conf.epp',
-      master_host            => $master_host,
-      puppetdb_database_host => $puppetdb_database_replica_host,
-    )
-  }
+  $puppetdb_database_replica_pe_conf = epp('pe_xl/puppetdb_database-pe.conf.epp',
+    master_host            => $master_host,
+    puppetdb_database_host => $puppetdb_database_replica_host,
+  )
 
   # Upload the pe.conf files to the hosts that need them
   pe_xl::file_content_upload($master_pe_conf, '/tmp/pe.conf', $master_host)
   pe_xl::file_content_upload($puppetdb_database_pe_conf, '/tmp/pe.conf', $puppetdb_database_host)
-
-  if $ha {
-    pe_xl::file_content_upload($puppetdb_database_replica_pe_conf, '/tmp/pe.conf', $puppetdb_database_replica_host)
-  }
+  pe_xl::file_content_upload($puppetdb_database_replica_pe_conf, '/tmp/pe.conf', $ha_database_target)
 
   # Download the PE tarball and send it to the nodes that need it
   $pe_tarball_name     = "puppet-enterprise-${version}-el-7-x86_64.tar.gz"
   $local_tarball_path  = "${stagingdir}/${pe_tarball_name}"
   $upload_tarball_path = "/tmp/${pe_tarball_name}"
 
-  if $ha {
-    $retrieve_and_upload_hosts = [$master_host, $puppetdb_database_host, $puppetdb_database_replica_host]
-  } else {
-    $retrieve_and_upload_hosts = [$master_host, $puppetdb_database_host]
-  }
-
   run_plan('pe_xl::util::retrieve_and_upload',
-    nodes       => $retrieve_and_upload_hosts,
+    nodes       => $pe_installer_hosts,
     source      => "https://s3.amazonaws.com/pe-builds/released/${version}/puppet-enterprise-${version}-el-7-x86_64.tar.gz",
     local_path  => $local_tarball_path,
     upload_path => $upload_tarball_path,
@@ -155,18 +149,16 @@
       | HEREDOC
   )
 
-  if $ha {
-    run_task('pe_xl::mkdir_p_file', $puppetdb_database_replica_host,
-      path    => '/etc/puppetlabs/puppet/csr_attributes.yaml',
-      content => @("HEREDOC"),
-        ---
-        extension_requests:
-          ${pp_application}: "puppet"
-          ${pp_role}: "pe_xl::puppetdb_database"
-          ${pp_cluster}: "B"
-        | HEREDOC
-    )
-  }
+  run_task('pe_xl::mkdir_p_file', $ha_database_target,
+    path    => '/etc/puppetlabs/puppet/csr_attributes.yaml',
+    content => @("HEREDOC"),
+      ---
+      extension_requests:
+        ${pp_application}: "puppet"
+        ${pp_role}: "pe_xl::puppetdb_database"
+        ${pp_cluster}: "B"
+      | HEREDOC
+  )
 
   # Get the master installation up and running. The installer will
   # "fail" because PuppetDB can't start. That's expected.
@@ -182,36 +174,15 @@
   }
 
   # Configure autosigning for the puppetdb database hosts 'cause they need it
-  if $ha {
-    run_task('pe_xl::mkdir_p_file', $master_host,
-      path    => '/etc/puppetlabs/puppet/autosign.conf',
-      owner   => 'pe-puppet',
-      group   => 'pe-puppet',
-      mode    => '0644',
-      content => @("HEREDOC"),
-        ${puppetdb_database_host}
-        ${puppetdb_database_replica_host}
-        | HEREDOC
-    )
-  } else {
-    run_task('pe_xl::mkdir_p_file', $master_host,
-      path    => '/etc/puppetlabs/puppet/autosign.conf',
-      owner   => 'pe-puppet',
-      group   => 'pe-puppet',
-      mode    => '0644',
-      content => @("HEREDOC"),
-        ${puppetdb_database_host}
-        | HEREDOC
-    )
-  }
+  run_task('pe_xl::mkdir_p_file', $master_host,
+    path    => '/etc/puppetlabs/puppet/autosign.conf',
+    owner   => 'pe-puppet',
+    group   => 'pe-puppet',
+    mode    => '0644',
+    content => $database_hosts.reduce |$memo,$host| { "${host}\n${memo}" },
+  )
 
   # Run the PE installer on the puppetdb database hosts
-  if $ha {
-    $database_hosts = [$puppetdb_database_host, $puppetdb_database_replica_host]
-  } else {
-    $database_hosts = [$puppetdb_database_host]
-  }
-
   run_task('pe_xl::pe_install', $database_hosts,
     tarball => $upload_tarball_path,
     peconf  => '/tmp/pe.conf',
@@ -244,51 +215,38 @@
   )
 
   # Deploy the PE agent to all remaining hosts
-  if $ha {
-    run_task('pe_xl::agent_install', $master_replica_host,
-      server        => $master_host,
-      install_flags => [
-        '--puppet-service-ensure', 'stopped',
-        "main:dns_alt_names=${dns_alt_names_csv}",
-        'extension_requests:pp_application=puppet',
-        'extension_requests:pp_role=pe_xl::master',
-        'extension_requests:pp_cluster=B',
-      ],
-    )
+  run_task('pe_xl::agent_install', $ha_replica_target,
+    server        => $master_host,
+    install_flags => [
+      '--puppet-service-ensure', 'stopped',
+      "main:dns_alt_names=${dns_alt_names_csv}",
+      'extension_requests:pp_application=puppet',
+      'extension_requests:pp_role=pe_xl::master',
+      'extension_requests:pp_cluster=B',
+    ],
+  )
 
-    run_task('pe_xl::agent_install', $cm_cluster_a,
-      server        => $master_host,
-      install_flags => [
-        '--puppet-service-ensure', 'stopped',
-        "main:dns_alt_names=${dns_alt_names_csv}",
-        'extension_requests:pp_application=puppet',
-        'extension_requests:pp_role=pe_xl::compiler',
-        'extension_requests:pp_cluster=A',
-      ],
-    )
+  run_task('pe_xl::agent_install', $cm_cluster_a,
+    server        => $master_host,
+    install_flags => [
+      '--puppet-service-ensure', 'stopped',
+      "main:dns_alt_names=${dns_alt_names_csv}",
+      'extension_requests:pp_application=puppet',
+      'extension_requests:pp_role=pe_xl::compiler',
+      'extension_requests:pp_cluster=A',
+    ],
+  )
 
-    run_task('pe_xl::agent_install', $cm_cluster_b,
-      server        => $master_host,
-      install_flags => [
-        '--puppet-service-ensure', 'stopped',
-        "main:dns_alt_names=${dns_alt_names_csv}",
-        'extension_requests:pp_application=puppet',
-        'extension_requests:pp_role=pe_xl::compiler',
-        'extension_requests:pp_cluster=B',
-      ],
-    )
-  } else {
-    run_task('pe_xl::agent_install', $compiler_hosts,
-      server        => $master_host,
-      install_flags => [
-        '--puppet-service-ensure', 'stopped',
-        "main:dns_alt_names=${dns_alt_names_csv}",
-        'extension_requests:pp_application=puppet',
-        'extension_requests:pp_role=pe_xl::compiler',
-        'extension_requests:pp_cluster=A',
-      ],
-    )
-  }
+  run_task('pe_xl::agent_install', $cm_cluster_b,
+    server        => $master_host,
+    install_flags => [
+      '--puppet-service-ensure', 'stopped',
+      "main:dns_alt_names=${dns_alt_names_csv}",
+      'extension_requests:pp_application=puppet',
+      'extension_requests:pp_role=pe_xl::compiler',
+      'extension_requests:pp_cluster=B',
+    ],
+  )
 
   # Do a Puppet agent run to ensure certificate requests have been submitted
   # These runs will "fail", and that's expected.
diff --git a/plans/upgrade.pp b/plans/upgrade.pp
index a3df3905..0772566e 100644
--- a/plans/upgrade.pp
+++ b/plans/upgrade.pp
@@ -1,11 +1,10 @@
 # @summary Upgrade an Extra Large stack from one .z to the next
 #
 plan pe_xl::upgrade (
-  Boolean   $ha = true,
   String[1] $master_host,
   String[1] $puppetdb_database_host,
-  String[1] $master_replica_host,
-  String[1] $puppetdb_database_replica_host,
+  Optional[String[1]] $master_replica_host = undef,
+  Optional[String[1]] $puppetdb_database_replica_host = undef,
 
   String[1] $version = '2018.1.4',
 
@@ -13,6 +12,14 @@
   String[1] $pe_source  = "https://s3.amazonaws.com/pe-builds/released/${version}/puppet-enterprise-${version}-el-7-x86_64.tar.gz",
 ) {
 
+  $ha_replica_target = [
+    $master_replica_host,
+  ].pe_xl::flatten_compact()
+
+  $ha_database_target = [
+    $puppetdb_database_replica_host,
+  ].pe_xl::flatten_compact()
+
   # Look up which hosts are compilers in the stack
   # We look up groups of CMs separately since when they are upgraded is determined
   # by which PDB PG host they are affiliated with
@@ -24,32 +31,22 @@
       !(certname = "${master_host}") }
     | PQL
 
-  if $ha {
-    $compiler_cluster_master_replica_hosts = puppetdb_query(@("PQL")).map |$node| { $node['certname'] }
-      resources[certname] { 
-        type = "Class" and
-        title = "Puppet_enterprise::Profile::Puppetdb" and
-        parameters.database_host = "${puppetdb_database_replica_host}" and
-        !(certname = "${master_replica_host}") }
-      | PQL
-  }
+  $compiler_cluster_master_replica_hosts = puppetdb_query(@("PQL")).map |$node| { $node['certname'] }
+    resources[certname] { 
+      type = "Class" and
+      title = "Puppet_enterprise::Profile::Puppetdb" and
+      parameters.database_host = "${puppetdb_database_replica_host}" and
+      !(certname = "${master_replica_host}") }
+    | PQL
 
-  if $ha {
-    $all_hosts = [
-      $master_host,
-      $puppetdb_database_host,
-      $master_replica_host,
-      $puppetdb_database_replica_host,
-      $compiler_cluster_master_hosts,
-      $compiler_cluster_master_replica_hosts,
-    ].pe_xl::flatten_compact()
-  } else {
-    $all_hosts = [
-      $master_host,
-      $puppetdb_database_host,
-      $compiler_cluster_master_hosts,
-    ].pe_xl::flatten_compact()
-  }
+  $all_hosts = [
+    $master_host,
+    $puppetdb_database_host,
+    $master_replica_host,
+    $puppetdb_database_replica_host,
+    $compiler_cluster_master_hosts,
+    $compiler_cluster_master_replica_hosts,
+  ].pe_xl::flatten_compact()
 
   $master_local = "local://${master_host}"
 
@@ -58,18 +55,11 @@
   # Download the PE tarball on the nodes that need it
   $upload_tarball_path = "/tmp/puppet-enterprise-${version}-el-7-x86_64.tar.gz"
 
-  if $ha {
-    $download_hosts = [
-      $master_host,
-      $puppetdb_database_host,
-      $puppetdb_database_replica_host,
-    ].pe_xl::flatten_compact()
-  } else {
-    $download_hosts = [
-      $master_host,
-      $puppetdb_database_host,
-    ].pe_xl::flatten_compact()
-  }
+  $download_hosts = [
+    $master_host,
+    $puppetdb_database_host,
+    $puppetdb_database_replica_host,
+  ].pe_xl::flatten_compact()
 
   run_task('pe_xl::download', $download_hosts,
     source => $pe_source,
@@ -137,29 +127,27 @@
     server => $master_host,
   )
 
-  if $ha {
-    # Shut down PuppetDB on CMs that use the PMR's PDB PG
-    run_task('service', $compiler_cluster_master_replica_hosts,
-      action => 'stop',
-      name   => 'pe-puppetdb',
-    )
+  # Shut down PuppetDB on CMs that use the PMR's PDB PG
+  run_task('service', $compiler_cluster_master_replica_hosts,
+    action => 'stop',
+    name   => 'pe-puppetdb',
+  )
 
-    # Run the upgrade.sh script on the master replica host
-    run_task('pe_xl::agent_upgrade', $master_replica_host,
-      server => $master_host,
-    )
+  # Run the upgrade.sh script on the master replica host
+  run_task('pe_xl::agent_upgrade', $ha_replica_target,
+    server => $master_host,
+  )
 
-    # Upgrade the master replica's PuppetDB PostgreSQL host
-    run_task('pe_xl::pe_install', $puppetdb_database_replica_host,
-      tarball => $upload_tarball_path,
-    )
-    run_task('pe_xl::puppet_runonce', $puppetdb_database_replica_host)
+  # Upgrade the master replica's PuppetDB PostgreSQL host
+  run_task('pe_xl::pe_install', $ha_database_target,
+    tarball => $upload_tarball_path,
+  )
+  run_task('pe_xl::puppet_runonce', $ha_database_target)
 
-    # Upgrade the compiler group B hosts
-    run_task('pe_xl::agent_upgrade', $compiler_cluster_master_replica_hosts,
-      server => $master_host,
-    )
-  }
+  # Upgrade the compiler group B hosts
+  run_task('pe_xl::agent_upgrade', $compiler_cluster_master_replica_hosts,
+    server => $master_host,
+  )
 
   # Ensure Puppet running on all infrastructure hosts
   run_task('service', $all_hosts,
diff --git a/templates/puppetdb_database-pe.conf.epp b/templates/puppetdb_database-pe.conf.epp
index 53036367..deab9e69 100644
--- a/templates/puppetdb_database-pe.conf.epp
+++ b/templates/puppetdb_database-pe.conf.epp
@@ -1,5 +1,5 @@
 <%- | String[1] $master_host,
-      String[1] $puppetdb_database_host,
+      Optional[String[1]] $puppetdb_database_host,
 | -%>
 #----------------------------------------------------------------------------
 # Puppet Enterprise installer configuration file

From 759fffb3431ef8d34cce109c4329f554dc6fbb5c Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppetlabs.com>
Date: Tue, 30 Jul 2019 17:42:32 -0700
Subject: [PATCH 09/15] Swich from notice() to out::message()

Newer versions of Bolt don't print anything to the screen when running
the notice() function. The replacement is out::message().
---
 plans/install.pp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/plans/install.pp b/plans/install.pp
index 31826fdb..498a13a5 100644
--- a/plans/install.pp
+++ b/plans/install.pp
@@ -163,14 +163,14 @@
   # Get the master installation up and running. The installer will
   # "fail" because PuppetDB can't start. That's expected.
   without_default_logging() || {
-    notice("Starting: task pe_xl::pe_install on ${master_host}")
+    out::message("Starting: task pe_xl::pe_install on ${master_host}")
     run_task('pe_xl::pe_install', $master_host,
       _catch_errors         => true,
       tarball               => $upload_tarball_path,
       peconf                => '/tmp/pe.conf',
       shortcircuit_puppetdb => true,
     )
-    notice("Finished: task pe_xl::pe_install on ${master_host}")
+    out::message("Finished: task pe_xl::pe_install on ${master_host}")
   }
 
   # Configure autosigning for the puppetdb database hosts 'cause they need it
@@ -251,9 +251,9 @@
   # Do a Puppet agent run to ensure certificate requests have been submitted
   # These runs will "fail", and that's expected.
   without_default_logging() || {
-    notice("Starting: task pe_xl::puppet_runonce on ${agent_installer_hosts}")
+    out::message("Starting: task pe_xl::puppet_runonce on ${agent_installer_hosts}")
     run_task('pe_xl::puppet_runonce', $agent_installer_hosts, {_catch_errors => true})
-    notice("Finished: task pe_xl::puppet_runonce on ${agent_installer_hosts}")
+    out::message("Finished: task pe_xl::puppet_runonce on ${agent_installer_hosts}")
   }
 
   # Ensure some basic configuration on the master needed at install time.

From 61ca327ec3e2fb798a9fdc09052ceee07820ea82 Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppetlabs.com>
Date: Wed, 31 Jul 2019 11:47:06 -0700
Subject: [PATCH 10/15] Handle undef in configure_node_groups task

So that when/if no HA systems are passed, the manifest works correctly.

This was a problem in the shell wrapper, not the Puppet code itself.
---
 tasks/configure_node_groups.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tasks/configure_node_groups.sh b/tasks/configure_node_groups.sh
index 0a1a9662..2c1fac6c 100755
--- a/tasks/configure_node_groups.sh
+++ b/tasks/configure_node_groups.sh
@@ -3,7 +3,12 @@
 
 /opt/puppetlabs/bin/puppet apply --environment production <<'EOF'
 
-function param($name) { inline_template("<%= ENV['PT_${name}'] %>") }
+function param($name) {
+  ($var = inline_template("<%= ENV['PT_${name}'] %>")) ? {
+    ''      => undef,
+    default => $var,
+  }
+}
 
 class configure_node_groups (
   String[1] $master_host            = param('master_host'),

From 5d6cf0ffa67ae8e4f0bb740d97addbfa0e0e1bc0 Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppetlabs.com>
Date: Wed, 31 Jul 2019 11:53:10 -0700
Subject: [PATCH 11/15] Add missing EOF to shell heredoc

Wasn't functionally necessary but this is more correct
---
 tasks/configure_node_groups.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tasks/configure_node_groups.sh b/tasks/configure_node_groups.sh
index 2c1fac6c..20870b62 100755
--- a/tasks/configure_node_groups.sh
+++ b/tasks/configure_node_groups.sh
@@ -167,3 +167,4 @@ class configure_node_groups (
   }
 
 }
+EOF

From 40d05f29fabd56d4a04be3be909c25bd75c89250 Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppetlabs.com>
Date: Wed, 31 Jul 2019 11:54:27 -0700
Subject: [PATCH 12/15] Add missing `include`

Not sure where this got lost, but it's a non-operational task without
it...
---
 tasks/configure_node_groups.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tasks/configure_node_groups.sh b/tasks/configure_node_groups.sh
index 20870b62..7b4b98a7 100755
--- a/tasks/configure_node_groups.sh
+++ b/tasks/configure_node_groups.sh
@@ -167,4 +167,6 @@ class configure_node_groups (
   }
 
 }
+
+include configure_node_groups
 EOF

From 4deccdf25c8c1b0a92a008cdcb10165c691c4fb6 Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppetlabs.com>
Date: Wed, 31 Jul 2019 12:58:13 -0700
Subject: [PATCH 13/15] Fix incorrect pe-installer/agent assignment

The database systems need to use the installer to set up, but the
replica doesn't.
---
 plans/install.pp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plans/install.pp b/plans/install.pp
index 498a13a5..2f75219e 100644
--- a/plans/install.pp
+++ b/plans/install.pp
@@ -56,7 +56,7 @@
   $pe_installer_hosts = [
     $master_host,
     $puppetdb_database_host,
-    $master_replica_host,
+    $puppetdb_database_replica_host,
   ].pe_xl::flatten_compact()
 
   $agent_installer_hosts = [

From deae78b1d12c2456aef8e1887301dcf089ec1fea Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppetlabs.com>
Date: Thu, 1 Aug 2019 09:45:03 -0700
Subject: [PATCH 14/15] Add "pe_master" variable to PE Master group

This identifier can help user code know whether or not a node is a
master, has resources like Service[pe-nginx], and so make intelligent
decisions about whether or not to apply addendum configuration that
depends on a node being a master.
---
 manifests/setup/node_manager.pp | 7 ++++---
 tasks/configure_node_groups.sh  | 7 ++++---
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp
index e0791dad..e7cee9b8 100644
--- a/manifests/setup/node_manager.pp
+++ b/manifests/setup/node_manager.pp
@@ -50,14 +50,15 @@
   # Because the group does not have any data by default this does not impact
   # out-of-box configuration of the group.
   node_group { 'PE Master':
-    parent => 'PE Infrastructure',
-    rule   => ['or',
+    parent    => 'PE Infrastructure',
+    rule      => ['or',
       ['and', ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::compiler']],
       ['=', 'name', $master_host],
     ],
-    data   => {
+    data      => {
       'pe_repo' => { 'compile_master_pool_address' => $compiler_pool_address },
     },
+    variables => { 'pe_master' => true },
   }
 
   # This class has to be included here because puppet_enterprise is declared
diff --git a/tasks/configure_node_groups.sh b/tasks/configure_node_groups.sh
index 7b4b98a7..5878f3e4 100755
--- a/tasks/configure_node_groups.sh
+++ b/tasks/configure_node_groups.sh
@@ -49,14 +49,15 @@ class configure_node_groups (
   # Because the group does not have any data by default this does not impact
   # out-of-box configuration of the group.
   node_group { 'PE Master':
-    parent => 'PE Infrastructure',
-    rule   => ['or',
+    parent    => 'PE Infrastructure',
+    rule      => ['or',
       ['and', ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::compiler']],
       ['=', 'name', $master_host],
     ],
-    data   => {
+    data      => {
       'pe_repo' => { 'compile_master_pool_address' => $compiler_pool_address },
     },
+    variables => { 'pe_master' => true },
   }
 
   # This class has to be included here because puppet_enterprise is declared

From e20788f4431d5bb9ae2f8ff1d6bb274baf74299a Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppetlabs.com>
Date: Thu, 1 Aug 2019 10:44:57 -0700
Subject: [PATCH 15/15] Update non-HA install doc

---
 documentation/install_and_configure_without_ha.md | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/documentation/install_and_configure_without_ha.md b/documentation/install_and_configure_without_ha.md
index ea0df2f7..610f6e0c 100644
--- a/documentation/install_and_configure_without_ha.md
+++ b/documentation/install_and_configure_without_ha.md
@@ -3,7 +3,7 @@
 * TODO: add this doc as a section to basic_usage.md instead?
 
 Please see the [basic_usage.md](basic_usage.md) document for reference; this document will avoid repeating the information covered there.
-The install, configure, and upgrade plans covered in the [basic_usage.md](basic_usage.md) document can also set up the Extra Large environment without HA by setting the optional `ha` parameter to `false` in the params.json file (see the [example](#example-params.json-bolt-parameters-file) below).
+The install, configure, and upgrade plans covered in the [basic_usage.md](basic_usage.md) document can also set up the Extra Large environment without HA by omitting the optional settings `master_replica_host` and `puppetdb_database_replica_host` in the params.json file (see the [example](#example-params.json-bolt-parameters-file) below).
 
 ## Basic usage instructions
 
@@ -11,7 +11,7 @@ The install, configure, and upgrade plans covered in the [basic_usage.md](basic_
 2. Install Bolt on a jumphost. This can be the master, or any other system.
 3. Download or git clone the pe\_xl module and put it somewhere on the jumphost, e.g. ~/modules/pe\_xl.
 4. Create an inventory file with connection information. An example is included below.
-5. Create a parameters file. An example is included below. Note the addition of the "ha" parameter with a value of "false", and the omission of the "replica" roles.
+5. Create a parameters file. An example is included below. Note the omission of the `master_replica_host` and `puppetdb_database_replica_host` parameters.
 6. Run the pe\_xl plan with the inputs created. Example:
 ```
         bolt plan run pe_xl \
@@ -48,12 +48,9 @@ groups:
   "install": true,
   "configure": true,
   "upgrade": false,
-  "ha": false,
 
   "master_host": "pe-xl-core-0.lab1.puppet.vm",
   "puppetdb_database_host": "pe-xl-core-1.lab1.puppet.vm",
-  "master_replica_host": "",
-  "puppetdb_database_replica_host": "",
   "compiler_hosts": [
     "pe-xl-compiler-0.lab1.puppet.vm",
     "pe-xl-compiler-1.lab1.puppet.vm"