.sync.yml

@@ -1,9 +1,14 @@
 ---
+Gemfile:
+  ':development':
+     required:
+       - gem: 'puppet-debugger'
+         version: '>= 0.18.0'
 Rakefile:
   extras:
     - 'PuppetSyntax.exclude_paths = ["plans/**/*.pp", "vendor/**/*"]'
 .gitignore:
   paths:
     - '.rerun.json'
     - '*.tar.gz'
-
+

CHANGELOG.md

@@ -1,3 +1,19 @@
+## Release 1.2.0
+### Summary
+
+Feature and bugfix release.
+
+### Features
+- Add direct download option for PE installers (download\_mode parameter)
+- Add docker features for testing deployments in containers
+- Improve idempotency around CSR submission and signing
+- Add basic version validation
+
+### Bugfixes
+- Make peadm::read\_file compatible with python3 for better CentOS 8 support
+- Fix failure to install when passing passing r10k\_private\_key parameters
+- Improve error handling of peadm::download task
+
 ## Release 1.1.0
 ### Summary
 

Gemfile

@@ -17,6 +17,7 @@ ruby_version_segments = Gem::Version.new(RUBY_VERSION.dup).segments
 minor_version = ruby_version_segments[0..1].join('.')
 
 group :development do
+  gem 'puppet-debugger', '0.18.0'
   gem "fast_gettext", '1.1.0',                                   require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.1.0')
   gem "fast_gettext",                                            require: false if Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.1.0')
   gem "json_pure", '<= 2.0.1',                                   require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.0.0')

README.md

@@ -17,9 +17,11 @@ Plans:
 * [Provision](documentation/provision.md)
 * [Upgrade](documentation/upgrade.md)
 * [Convert](documentation/convert.md)
+* [Status](documentation/status.md)
 
 Reference:
 
+* [PE Architecture Documentation](https://puppet.com/docs/pe/latest/choosing_an_architecture.html)
 * [Classification](documentation/classification.md)
 * [Architectures](documentation/architectures.md)
 * [Testing](documentation/pre_post_checks.md)
@@ -30,3 +32,7 @@ Reference:
 Normally, if you are able to use peadm to set up an infrastructure and Puppet agent runs are all working, chances are you met all the requirements and don't have to worry. Sometimes Some notable requirements are highlighted below which may accidentally be adjusted by users, but which architectures deployed by this module rely on. These configuration requirements need to be maintained for the infrastructure to operate correctly.
 
 * Classifier Data needs to be enabled. This feature is enabled by default on new installs, but can be disabled by users if they remove the relevant configuration from their global hiera.yaml file. See the [PE docs](https://puppet.com/docs/pe/latest/config_console.html#task-5039) for more information.
+
+## Getting Help
+
+To get help with issues concerning this module, please make use of [issues](https://github.com/puppetlabs/puppetlabs-peadm/issues) in the project on GitHub.

documentation/docker_examples.md

@@ -69,6 +69,54 @@ This will run an interactive bash shell in the running container.
 ### Upgrades
 There is also a upgrade.sh script that is similar to the provision.sh script.  This script will upgrade an already provisioned PE stack to the version specified in the update_params.json file.
 
+## Attaching agent containers
+Because we are using containers we can start up numerous container instances and attach them all to the same PE stack.  In a matter of minutes you can easily attach hunders of new nodes to the PE stack (if resources allow). This is very useful for testing out scenarios involving different puppet versions or operating systems and even features like the orchestrator. 
+
+To attach a container to the PE stack you first must get the network name of the PE stack.  This can be done with a command like: `docker inspect pe-xl-core-0.puppet.vm`.  You need to use either the container id or the container name of the MoM when inspecting. 
+
+The network name we want to grab is 'extra-large-ha_default'.  (Yours will be different, but the output will be similar)
+
+```shell
+docker inspect pe-xl-core-0.puppet.vm -f "{{json .NetworkSettings.Networks }}" | jq
+{
+  "extra-large-ha_default": {
+    "IPAMConfig": null,
+    "Links": null,
+    "Aliases": [
+      "pe_xl_core_0",
+      "5cf7047a36cd"
+    ],
+    "NetworkID": "204ae562a25510b2425f9fe3f1599c487e40dbcaaaaf02c2f73f6fa81f45d674",
+    "EndpointID": "d91d7060fcc623a9f16cea09eecf83e9ee4454252e1af34053ef090f9c01c9c3",
+    "Gateway": "172.25.0.1",
+    "IPAddress": "172.25.0.6",
+    "IPPrefixLen": 16,
+    "IPv6Gateway": "",
+    "GlobalIPv6Address": "",
+    "GlobalIPv6PrefixLen": 0,
+    "MacAddress": "02:42:ac:19:00:06",
+    "DriverOpts": null
+  }
+}
+```
+
+**NOTE** In these example you may see the use of `jq`.  This is a [cli utility for parsing JSON](https://stedolan.github.io/jq/).  I recommend installing it.  As a alternative you can pipe output to `python -m json.tool`.
+
+### Starting agent containers
+Once you have the network name you only need to specify the network when starting a container.  Puppet [publishes container images](https://hub.docker.com/r/puppet/puppet-agent) for all version of the puppet agent.  So you can easily switch agent versions with a single command. Which container image you use is entirely up to you.  If it doesn't have puppet preinstalled you can use the special curl command from the PE console to install it. 
+
+Example:
+ `docker run -ti --network=extra-large-ha_default --entrypoint=/bin/bash puppet/puppet-agent:latest`
+ `docker run -ti --network=extra-large-ha_default --entrypoint=/bin/bash puppet/puppet-agent:6.15.0`
+ `docker run -ti --network=extra-large-ha_default --entrypoint=/bin/bash puppet/puppet-agent:6.3.0`
+ `docker run -ti --network=extra-large-ha_default --entrypoint=/bin/bash ruby:latest`
+
+For most tasks these images are great.  However, if you wish to use puppet orchestrator with the pcp transport. The one requirement is that all images used must be systemd aware, otherwise pxp will not start. If you do not plan on using pcp 
+there is no need for containers with systemd.
+
+At this time we have not added documention for starting a container with systemd.  Instructions coming soon.
+
+
 ### Other notes
 1. The provision plan is not fully idempotent.
 2. Some tasks may fail when run due to resource constraints.

documentation/provision.md

@@ -103,6 +103,10 @@ The content needed is the PE installation tarball for the target version. The in
 
 Installation content can be downloaded from [https://puppet.com/try-puppet/puppet-enterprise/download/](https://puppet.com/try-puppet/puppet-enterprise/download/).
 
+## Online usage
+
+The peadm::provision plan can be configured to download installation content directly to hosts. To configure online installation, set the `download_mode` parameter of the `peadm::provision` plan to `direct`. The direct mode is often more efficient when PE hosts have a route to the internet.
+
 ## Hostnames and Certificate Names
 
 The various host parameters given to the peadm::provision or peadm::action::install plans will be set as Puppet certificate names. You must use the names here that you want the servers to be identified as by Puppet.

documentation/upgrade.md

@@ -40,6 +40,10 @@ The content needed is the PE installation tarball for the target version. The in
 
 Installation content can be downloaded from [https://puppet.com/try-puppet/puppet-enterprise/download/](https://puppet.com/try-puppet/puppet-enterprise/download/).
 
+## Online usage
+
+The peadm::provision plan can be configured to download installation content directly to hosts. To configure online installation, set the `download_mode` parameter of the `peadm::provision` plan to `direct`. The direct mode is often more efficient when PE hosts have a route to the internet.
+
 ## Usage over the Orchestrator transport
 
 The peadm::upgrade plan can be used with the Orchestrator (pcp) transport, provided that the Bolt executor is running as root on the master. To use the Orchestrator transport prepare an inventory file such as the following to set the default transport to be `pcp`, but the master specifically to be `local`.

functions/validate_version.pp

@@ -0,0 +1,18 @@
+function peadm::validate_version(
+  String $version,
+) {
+  $supported = ($version =~ SemVerRange('>= 2019.1.0 <= 2019.5.0'))
+
+  unless $supported {
+    fail(@("REASON"/L))
+      This version of the puppetlabs-peadm module does not support PE ${version}.
+
+      For PE versions older than 2019.1, please use version 0.4.x of the \
+      puppetlabs-pe_xl module.
+
+      For PE versions 2019.7 and newer, check to see if a new version of peadm \
+      exists which supports that version of PE.
+
+      | REASON
+  }
+}

metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "puppetlabs-peadm",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "author": "Puppet Labs Solutions Architecture",
   "summary": "Bolt plans used to deploy an at-scale Puppet Enterprise architecture",
   "license": "Apache-2.0",

plans/action/install.pp

@@ -46,8 +46,11 @@
   Optional[String]     $license_key_content = undef,
 
   # Other
-  String               $stagingdir   = '/tmp',
+  String                $stagingdir    = '/tmp',
+  Enum[direct,bolthost] $download_mode = 'bolthost',
 ) {
+  peadm::validate_version($version)
+
   # Convert inputs into targets.
   $master_target                    = peadm::get_targets($master_host, 1)
   $master_replica_target            = peadm::get_targets($master_replica_host, 1)
@@ -168,16 +171,24 @@
     )
   }
 
-  # Download the PE tarball and send it to the nodes that need it
   $pe_tarball_name     = "puppet-enterprise-${version}-${platform}.tar.gz"
-  $local_tarball_path  = "${stagingdir}/${pe_tarball_name}"
+  $pe_tarball_source   = "https://s3.amazonaws.com/pe-builds/released/${version}/${pe_tarball_name}"
   $upload_tarball_path = "/tmp/${pe_tarball_name}"
 
-  run_plan('peadm::util::retrieve_and_upload', $pe_installer_targets,
-    source      => "https://s3.amazonaws.com/pe-builds/released/${version}/puppet-enterprise-${version}-${platform}.tar.gz",
-    local_path  => $local_tarball_path,
-    upload_path => $upload_tarball_path,
-  )
+  if $download_mode == 'bolthost' {
+    # Download the PE tarball and send it to the nodes that need it
+    run_plan('peadm::util::retrieve_and_upload', $pe_installer_targets,
+      source      => $pe_tarball_source,
+      local_path  => "${stagingdir}/${pe_tarball_name}",
+      upload_path => $upload_tarball_path,
+    )
+  } else {
+    # Download PE tarballs directly to nodes that need it
+    run_task('peadm::download', $pe_installer_targets,
+      source => $pe_tarball_source,
+      path   => $upload_tarball_path,
+    )
+  }
 
   # Create csr_attributes.yaml files for the nodes that need them. Ensure that
   # if a csr_attributes.yaml file is already present, the values we need are
@@ -226,8 +237,6 @@
       $master_replica_target,
     ]),
       path    => '/etc/puppetlabs/puppetserver/ssh/id-control_repo.rsa',
-      owner   => 'pe-puppet',
-      group   => 'pe-puppet',
       mode    => '0400',
       content => $r10k_private_key,
     )
@@ -239,8 +248,6 @@
       $master_replica_target,
     ]),
       path    => '/etc/puppetlabs/license.key',
-      owner   => 'pe-puppet',
-      group   => 'pe-puppet',
       mode    => '0400',
       content => $license_key,
     )

plans/provision.pp

@@ -32,8 +32,10 @@
   Optional[String]                  $license_key_content = undef,
 
   # Other
-  Optional[String]                  $stagingdir = undef,
+  Optional[String]                  $stagingdir    = undef,
+  Enum[direct,bolthost]             $download_mode = 'bolthost',
 ) {
+  peadm::validate_version($version)
 
   $install_result = run_plan('peadm::action::install',
     # Standard
@@ -64,6 +66,7 @@
 
     # Other
     stagingdir                     => $stagingdir,
+    download_mode                  => $download_mode,
   )
 
   $configure_result = run_plan('peadm::action::configure',

plans/upgrade.pp

@@ -16,8 +16,11 @@
   String $version,
 
   # Other
-  String[1] $stagingdir = '/tmp',
+  String                $stagingdir    = '/tmp',
+  Enum[direct,bolthost] $download_mode = 'bolthost',
 ) {
+  peadm::validate_version($version)
+
   # Ensure input valid for a supported architecture
   $arch = peadm::validate_architecture(
     $master_host,
@@ -82,16 +85,25 @@
   # if the orchestrator is being used for the master.
   $master_target.peadm::fail_on_transport('pcp')
 
-  # Download the PE tarball on the nodes that need it
   $platform = run_task('peadm::precheck', $master_target).first['platform']
   $tarball_filename = "puppet-enterprise-${version}-${platform}.tar.gz"
+  $tarball_source   = "https://s3.amazonaws.com/pe-builds/released/${version}/${tarball_filename}"
   $upload_tarball_path = "/tmp/${tarball_filename}"
 
-  run_plan('peadm::util::retrieve_and_upload', $pe_installer_targets,
-    source      => "https://s3.amazonaws.com/pe-builds/released/${version}/${tarball_filename}",
-    local_path  => "${stagingdir}/${tarball_filename}",
-    upload_path => $upload_tarball_path,
-  )
+  if $download_mode == 'bolthost' {
+    # Download the PE tarball on the nodes that need it
+    run_plan('peadm::util::retrieve_and_upload', $pe_installer_targets,
+      source      => $tarball_source,
+      local_path  => "${stagingdir}/${tarball_filename}",
+      upload_path => $upload_tarball_path,
+    )
+  } else {
+    # Download PE tarballs directly to nodes that need it
+    run_task('peadm::download', $pe_installer_targets,
+      source => $tarball_source,
+      path   => $upload_tarball_path,
+    )
+  }
 
   # Shut down Puppet on all infra targets
   run_task('service', $all_targets,

spec/docker/Dockerfile

@@ -26,6 +26,18 @@ RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == system
   rm -rf /var/cache/yum; \
   ln -s '/usr/lib/systemd/system/sshd.service' '/etc/systemd/system/multi-user.target.wants/sshd.service' && \
   ln -s '/etc/systemd/system/live_audit.service' '/etc/systemd/system/multi-user.target.wants/live_audit.service'
+ARG HOST="localhost"
+RUN mkdir -p /root/.puppetlabs/client-tools && \
+    echo $' \n\
+{ \n\
+  "puppetdb": { \n\
+    "server_urls": "https://'$HOST$':8081", \n\
+    "cacert": "/etc/puppetlabs/puppet/ssl/certs/ca.pem", \n\
+    "cert": "/etc/puppetlabs/puppet/ssl/certs/'$HOST$'.pem", \n\
+    "key": "/etc/puppetlabs/puppet/ssl/private_keys/'$HOST$'.pem" \n\
+  } \n\
+} \n '\
+> /root/.puppetlabs/client-tools/puppetdb.conf
 VOLUME [ “/sys/fs/cgroup” ]
 #CMD /bin/bash
 ENTRYPOINT [ "/sbin/init" ]

spec/docker/Dockerfile_bolt

@@ -1,7 +1,10 @@
 FROM puppet/puppet-bolt
-ENV LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8"
-RUN echo "LANG=en_US.UTF-8" > /etc/locale.conf
-RUN apt-get update && apt-get install -y ssh sudo curl; \
+RUN apt-get update && apt-get install -y locales ssh sudo curl; \
    /opt/puppetlabs/bolt/bin/gem install bundler puppet-debugger -N -q
+ENV LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8"
+RUN echo "LC_ALL=en_US.UTF-8" >> /etc/environment && \
+    echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \
+    echo "LANG=en_US.UTF-8" > /etc/locale.conf && \
+    locale-gen en_US.UTF-8
 CMD /bin/bash
 ENTRYPOINT [ "/opt/puppetlabs/bin/bolt" ]

spec/docker/extra-large-ha/docker-compose.yaml

@@ -13,6 +13,7 @@ services:
     depends_on:
       - pe_xl_core_0
   compiler1:
+    restart: always
     depends_on:
       - pe_xl_core_0
     build:
@@ -33,6 +34,7 @@ services:
     volumes:
       - '/sys/fs/cgroup:/sys/fs/cgroup:ro'  
   pe_pdb:
+    restart: always
     depends_on:
       - pe_xl_core_0
     build:
@@ -52,6 +54,7 @@ services:
     volumes:
       - '/sys/fs/cgroup:/sys/fs/cgroup:ro' 
   pe_pdb-replica:
+    restart: always
     depends_on:
       - pe_xl_core_0
     build:
@@ -71,11 +74,14 @@ services:
     volumes:
       - '/sys/fs/cgroup:/sys/fs/cgroup:ro'       
   pe_xl_core_1:
+    restart: always
     depends_on:
       - pe_xl_core_0
     build:
       dockerfile: 'Dockerfile'
       context: ../
+      args:
+        HOST: 'pe-xl-core-1.puppet.vm'
     entrypoint: /sbin/init
     image: pe-base
     privileged: true # required for systemd
@@ -96,9 +102,12 @@ services:
     volumes:
       - '/sys/fs/cgroup:/sys/fs/cgroup:ro'      
   pe_xl_core_0:
+    restart: always
     build:
       dockerfile: 'Dockerfile'
       context: ../
+      args:
+        HOST: 'pe-xl-core-0.puppet.vm'
     entrypoint: /sbin/init
     image: pe-base
     privileged: true # required for systemd

spec/docker/extra-large/docker-compose.yaml

@@ -55,6 +55,8 @@ services:
     build:
       dockerfile: 'Dockerfile'
       context: ../
+      args:
+        HOST: 'pe-xl-core-0.puppet.vm'
     entrypoint: /sbin/init
     image: pe-base
     privileged: true # required for systemd