Showing with 5,198 additions and 1,293 deletions.
  1. +17 −0 .fixtures.yml
  2. +2 −0 .gitignore
  3. +8 −5 .gitlab-ci.yml
  4. +11 −0 .rubocop.yml
  5. +16 −0 .sync.yml
  6. +8 −4 .travis.yml
  7. +1 −1 .vscode/extensions.json
  8. +67 −0 CHANGELOG.md
  9. +6 −4 Gemfile
  10. +5 −0 Puppetfile
  11. +29 −7 README.md
  12. +3 −0 Rakefile
  13. +15 −0 documentation/architectures.md
  14. +0 −76 documentation/basic_usage.md
  15. +26 −0 documentation/convert.md
  16. +128 −0 documentation/docker_examples.md
  17. BIN documentation/images/table_output.png
  18. +0 −60 documentation/install_and_configure_without_ha.md
  19. +0 −67 documentation/large_deploy.md
  20. +146 −0 documentation/pre_post_checks.md
  21. +151 −0 documentation/provision.md
  22. +170 −0 documentation/res/raw_summary.json
  23. +111 −0 documentation/res/summarized.json
  24. +166 −0 documentation/status.md
  25. +132 −0 documentation/upgrade.md
  26. +19 −0 examples/provision/extra-large-ha.json
  27. +19 −0 examples/provision/extra-large.json
  28. +19 −0 examples/provision/large-ha.json
  29. +19 −0 examples/provision/large.json
  30. +5 −0 examples/provision/minimal.json
  31. +19 −0 examples/provision/standard-ha.json
  32. +19 −0 examples/provision/standard.json
  33. +2 −0 examples/ruby_task.rb
  34. +13 −0 functions/convert_hash.pp
  35. +50 −0 functions/convert_status.pp
  36. +62 −0 functions/determine_status.pp
  37. +1 −1 functions/fail_on_transport.pp
  38. +19 −0 functions/file_or_content.pp
  39. +1 −1 functions/flatten_compact.pp
  40. +1 −1 functions/generate_pe_conf.pp
  41. +1 −1 functions/get_targets.pp
  42. +12 −0 functions/oid.pp
  43. +0 −20 functions/print_apply_result.pp
  44. +6 −3 functions/{target_host.pp → target_name.pp}
  45. +2 −2 functions/validate_architecture.pp
  46. +18 −0 functions/validate_version.pp
  47. +16 −0 functions/wait_until_service_ready.pp
  48. +0 −4 installer/primary-master/csr_attributes.yaml
  49. +0 −64 installer/primary-master/pe.conf
  50. +0 −4 installer/puppetdb-database/csr_attributes.yaml
  51. +0 −21 installer/puppetdb-database/pe.conf
  52. +4 −2 lib/puppet/functions/{pe_xl → peadm}/file_content_upload.rb
  53. +10 −0 lib/puppet/functions/peadm/node_manager_yaml_location.rb
  54. +1 −1 manifests/compiler.pp
  55. +1 −1 manifests/load_balancer.pp
  56. +1 −1 manifests/puppetdb.pp
  57. +26 −0 manifests/setup/convert_pe2018.pp
  58. +50 −57 manifests/setup/node_manager.pp
  59. +16 −0 manifests/setup/node_manager_yaml.pp
  60. +24 −15 metadata.json
  61. +115 −0 plans/action/configure.pp
  62. +379 −0 plans/action/install.pp
  63. +173 −0 plans/convert.pp
  64. +2 −2 plans/misc/divert_code_manager.pp
  65. +22 −10 plans/provision.pp
  66. +99 −0 plans/status.pp
  67. +0 −117 plans/unit/configure.pp
  68. +0 −310 plans/unit/install.pp
  69. +203 −130 plans/upgrade.pp
  70. +111 −0 plans/util/add_cert_extensions.pp
  71. +27 −0 plans/util/insert_csr_extension_requests.pp
  72. +0 −29 plans/util/install_module.pp
  73. +4 −4 plans/util/retrieve_and_upload.pp
  74. +20 −0 plans/util/sanitize_pg_pe_conf.pp
  75. +2 −0 project.yaml
  76. +19 −0 provision.yaml
  77. +16 −0 spec/acceptance/peadm_spec.rb
  78. +1 −0 spec/default_facts.yml
  79. +1 −0 spec/docker/.dockerignore
  80. +43 −0 spec/docker/Dockerfile
  81. +127 −0 spec/docker/extra-large-ha/docker-compose.yaml
  82. +11 −0 spec/docker/extra-large-ha/params.json
  83. +8 −0 spec/docker/extra-large-ha/upgrade_params.json
  84. +72 −0 spec/docker/extra-large/docker-compose.yaml
  85. +8 −0 spec/docker/extra-large/params.json
  86. +6 −0 spec/docker/extra-large/upgrade_params.json
  87. +15 −0 spec/docker/inventory.yaml
  88. +80 −0 spec/docker/large-ha/docker-compose.yaml
  89. +8 −0 spec/docker/large-ha/params.json
  90. +7 −0 spec/docker/large-ha/upgrade_params.json
  91. +51 −0 spec/docker/large/docker-compose.yaml
  92. +7 −0 spec/docker/large/params.json
  93. +6 −0 spec/docker/large/upgrade_params.json
  94. +8 −0 spec/docker/live_audit.service
  95. +20 −0 spec/docker/live_audit.sh
  96. +31 −0 spec/docker/provision.sh
  97. +58 −0 spec/docker/standard-ha/docker-compose.yaml
  98. +7 −0 spec/docker/standard-ha/params.json
  99. +6 −0 spec/docker/standard-ha/upgrade_params.json
  100. +33 −0 spec/docker/standard/docker-compose.yaml
  101. +7 −0 spec/docker/standard/params.json
  102. +5 −0 spec/docker/standard/upgrade_params.json
  103. +31 −0 spec/docker/upgrade.sh
  104. +469 −0 spec/fixtures/infrastatus.json
  105. +12 −0 spec/fixtures/plans/failed_table.txt
  106. +16 −0 spec/fixtures/plans/passed_table.txt
  107. +170 −0 spec/fixtures/plans/raw_summary.json
  108. +111 −0 spec/fixtures/plans/summarized.json
  109. +10 −0 spec/fixtures/plans/summary_table.txt
  110. +41 −0 spec/fixtures/status.json
  111. +16 −0 spec/functions/convert_hash_spec.rb
  112. +17 −0 spec/functions/convert_status_spec.rb
  113. +14 −0 spec/functions/determine_status_spec.rb
  114. +14 −0 spec/functions/fail_on_transport_spec.rb
  115. +17 −0 spec/functions/file_or_content_spec.rb
  116. +11 −0 spec/functions/flatten_compact_spec.rb
  117. +32 −0 spec/functions/generate_pe_conf_spec.rb
  118. +16 −0 spec/functions/get_targets_spec.rb
  119. +16 −0 spec/functions/oid_spec.rb
  120. +14 −0 spec/functions/target_name_spec.rb
  121. +80 −0 spec/functions/validate_architecture_spec.rb
  122. +67 −0 spec/plans/status_spec.rb
  123. +3 −0 spec/spec_helper.rb
  124. +6 −0 spec/spec_helper_acceptance.rb
  125. +13 −0 spec/spec_helper_acceptance_local.rb
  126. +34 −0 spec/spec_helper_local.rb
  127. +2 −1 tasks/agent_install.json
  128. +1 −1 tasks/agent_install.sh
  129. +4 −0 tasks/agent_upgrade.sh
  130. +0 −29 tasks/configure_node_groups.json
  131. +0 −176 tasks/configure_node_groups.sh
  132. +2 −1 tasks/download.sh
  133. +2 −2 tasks/enable_replica.sh
  134. +10 −1 tasks/filesize.sh
  135. +0 −8 tasks/hostname.json
  136. +0 −6 tasks/hostname.sh
  137. +15 −0 tasks/infrastatus.json
  138. +16 −0 tasks/infrastatus.sh
  139. +17 −0 tasks/mv.json
  140. +11 −0 tasks/mv.sh
  141. +0 −8 tasks/orchestrator_healthcheck.json
  142. +0 −22 tasks/orchestrator_healthcheck.sh
  143. +4 −0 tasks/pe_install.json
  144. +10 −0 tasks/pe_install.sh
  145. +8 −0 tasks/precheck.json
  146. +18 −0 tasks/precheck.sh
  147. +5 −0 tasks/provision_replica.json
  148. +25 −9 tasks/provision_replica.sh
  149. +21 −0 tasks/puppet_infra_upgrade.json
  150. +28 −0 tasks/puppet_infra_upgrade.rb
  151. +2 −0 tasks/rbac_token.rb
  152. +12 −0 tasks/read_file.json
  153. +32 −0 tasks/read_file.sh
  154. +13 −0 tasks/sign_csr.json
  155. +33 −0 tasks/sign_csr.rb
  156. +10 −0 tasks/ssl_clean.json
  157. +29 −0 tasks/ssl_clean.rb
  158. +8 −0 tasks/submit_csr.json
  159. +41 −0 tasks/submit_csr.rb
  160. +8 −0 tasks/trusted_facts.json
  161. +42 −0 tasks/trusted_facts.rb
  162. +16 −0 tasks/wait_until_service_ready.json
  163. +17 −0 tasks/wait_until_service_ready.sh
  164. +8 −0 templates/node_manager.yaml.epp
  165. +1 −1 types/pem.pp
  166. +5 −3 types/singletargetspec.pp
17 changes: 17 additions & 0 deletions .fixtures.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
---
fixtures:
forge_modules:
ruby_task_helper: "puppetlabs/ruby_task_helper"
repositories:
facts: 'https://github.com/puppetlabs/puppetlabs-facts.git'
puppet_agent: 'https://github.com/puppetlabs/puppetlabs-puppet_agent.git'
provision: 'https://github.com/puppetlabs/provision.git'
stdlib: 'https://github.com/puppetlabs/puppetlabs-stdlib.git'
node_manager: 'https://github.com/WhatsARanjit/puppet-node_manager'
apply_helpers: 'https://github.com/puppetlabs/puppetlabs-apply_helpers'
bolt_shim: 'https://github.com/puppetlabs/puppetlabs-bolt_shim'
debug: 'https://github.com/nwops/puppet-debug'
format: 'https://github.com/voxpupuli/puppet-format'
container_inventory: 'https://gitlab.com/nwops/bolt-container_inventory'
symlinks:
"peadm": "#{source_dir}"
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,5 @@
.project
.envrc
/inventory.yaml
.rerun.json
*.tar.gz
13 changes: 8 additions & 5 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,25 @@ cache:
before_script:
- bundle -v
- rm Gemfile.lock || true
- gem update --system $RUBYGEMS_VERSION
- "# Update system gems if requested. This is useful to temporarily workaround troubles in the test runner"
- "# Set `rubygems_version` in the .sync.yml to set a value"
- "# Ignore exit code of SIGPIPE'd yes to not fail with shell's pipefail set"
- '[ -z "$RUBYGEMS_VERSION" ] || (yes || true) | gem update --system $RUBYGEMS_VERSION'
- gem --version
- bundle -v
- bundle install --without system_tests --path vendor/bundle --jobs $(nproc)

syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop-Ruby 2.5.3-Puppet ~> 6:
syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop-Ruby 2.5.7-Puppet ~> 6:
stage: syntax
image: ruby:2.5.3
image: ruby:2.5.7
script:
- bundle exec rake syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop
variables:
PUPPET_GEM_VERSION: '~> 6'

parallel_spec-Ruby 2.5.3-Puppet ~> 6:
parallel_spec-Ruby 2.5.7-Puppet ~> 6:
stage: unit
image: ruby:2.5.3
image: ruby:2.5.7
script:
- bundle exec rake parallel_spec
variables:
Expand Down
11 changes: 11 additions & 0 deletions .rubocop.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ GetText/DecorateString:
Description: We don't want to decorate test output.
Exclude:
- spec/**/*
Enabled: false
RSpec/BeforeAfterAll:
Description: Beware of using after(:all) as it may cause state to leak between tests.
A necessary evil in acceptance testing.
Expand All @@ -39,6 +40,10 @@ Style/BlockDelimiters:
Description: Prefer braces for chaining. Mostly an aesthetical choice. Better to
be consistent then.
EnforcedStyle: braces_for_chaining
Style/BracesAroundHashParameters:
Description: Braces are required by Ruby 2.7. Cop removed from RuboCop v0.80.0.
See https://github.com/rubocop-hq/rubocop/pull/7643
Enabled: true
Style/ClassAndModuleChildren:
Description: Compact style reduces the required amount of indentation.
EnforcedStyle: compact
Expand Down Expand Up @@ -88,6 +93,12 @@ Style/MethodCalledOnDoEndBlock:
Enabled: true
Style/StringMethods:
Enabled: true
GetText/DecorateFunctionMessage:
Enabled: false
GetText/DecorateStringFormattingUsingInterpolation:
Enabled: false
GetText/DecorateStringFormattingUsingPercent:
Enabled: false
Layout/EndOfLine:
Enabled: false
Layout/IndentHeredoc:
Expand Down
16 changes: 16 additions & 0 deletions .sync.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,20 @@
---
Gemfile:
required:
':development':
- gem: 'puppet-debugger'
version: '>= 0.18.0'
optional:
':development':
- gem: 'github_changelog_generator'
git: 'https://github.com/skywinder/github-changelog-generator'
ref: '20ee04ba1234e9e83eb2ffb5056e23d641c7a018'
condition: "Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.2.2')"
Rakefile:
changelog_since_tag: '2.1.0'
extras:
- 'PuppetSyntax.exclude_paths = ["plans/**/*.pp", "vendor/**/*"]'
.gitignore:
paths:
- '.rerun.json'
- '*.tar.gz'
12 changes: 8 additions & 4 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,26 +1,30 @@
---
os: linux
dist: xenial
language: ruby
cache: bundler
before_install:
- bundle -v
- rm -f Gemfile.lock
- gem update --system $RUBYGEMS_VERSION
- "# Update system gems if requested. This is useful to temporarily workaround troubles in the test runner"
- "# See https://github.com/puppetlabs/pdk-templates/commit/705154d5c437796b821691b707156e1b056d244f for an example of how this was used"
- "# Ignore exit code of SIGPIPE'd yes to not fail with shell's pipefail set"
- '[ -z "$RUBYGEMS_VERSION" ] || (yes || true) | gem update --system $RUBYGEMS_VERSION'
- gem --version
- bundle -v
script:
- 'bundle exec rake $CHECK'
bundler_args: --without system_tests
rvm:
- 2.5.3
- 2.5.7
stages:
- static
- spec
- acceptance
-
if: tag =~ ^v\d
name: deploy
matrix:
jobs:
fast_finish: true
include:
-
Expand All @@ -32,7 +36,7 @@ matrix:
stage: spec
-
env: PUPPET_GEM_VERSION="~> 6.0" CHECK=parallel_spec
rvm: 2.5.3
rvm: 2.5.7
stage: spec
-
env: DEPLOY_TO_FORGE=yes
Expand Down
2 changes: 1 addition & 1 deletion .vscode/extensions.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"recommendations": [
"jpogran.puppet-vscode",
"puppet.puppet-vscode",
"rebornix.Ruby"
]
}
67 changes: 67 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
## Release 2.1.1
### Summary

Development tool and README fixes.

### Bugfixes

- Remove reference to Puppet Support team from README. This module is intended to be used in collaboration with Professional Services and Solutions Architects at Puppet, not Support
- Fixes and improvements to Docker development tools

## Release 2.1.0
### Summary

Support upgrades from PE 2018.1 to 2019.7.

### Features

- Support added for upgrading from PE 2018.1 to 2019.7

## Release 2.0.0
### Summary

Major version release to support PE 2019.7.

Users can use peadm 2.0.0 to create new 2019.7 deployments, or to upgrade from
2019.5 to 2019.7.

To deploy PE 2019.5 or older, use a 1.x release of peadm.

### Features
- Support added for PE 2019.7

## Release 1.2.0
### Summary

Feature and bugfix release.

### Features
- Add direct download option for PE installers (download\_mode parameter)
- Add docker features for testing deployments in containers
- Improve idempotency around CSR submission and signing
- Add basic version validation

### Bugfixes
- Make peadm::read\_file compatible with python3 for better CentOS 8 support
- Fix failure to install when passing passing r10k\_private\_key parameters
- Improve error handling of peadm::download task

## Release 1.1.0
### Summary

This release supports PE 2019.1 through 2019.5.

A Changelog was not maintained prior to this release.

### Features
- Provision new PE clusters with standard, large, or extra-large architecture
- Upgrade PE clusters provisioned with peadm

### Bugfixes

N/A

This changelog is used track changes with this module in human readable format.
Feel free to reference tickets with links or other important information the
reader would find useful when determining the level of risk with upgrading.
For more information on changelogs please [see the keeping a changelog site](http://keepachangelog.com/en/0.3.0/).
10 changes: 6 additions & 4 deletions Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,12 @@ group :development do
gem "json", '= 2.0.4', require: false if Gem::Requirement.create('~> 2.4.2').satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
gem "json", '= 2.1.0', require: false if Gem::Requirement.create(['>= 2.5.0', '< 2.7.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
gem "rb-readline", '= 0.5.5', require: false, platforms: [:mswin, :mingw, :x64_mingw]
gem "puppet-module-posix-default-r#{minor_version}", '~> 0.3', require: false, platforms: [:ruby]
gem "puppet-module-posix-dev-r#{minor_version}", '~> 0.3', require: false, platforms: [:ruby]
gem "puppet-module-win-default-r#{minor_version}", '~> 0.3', require: false, platforms: [:mswin, :mingw, :x64_mingw]
gem "puppet-module-win-dev-r#{minor_version}", '~> 0.3', require: false, platforms: [:mswin, :mingw, :x64_mingw]
gem "puppet-module-posix-default-r#{minor_version}", '~> 0.4', require: false, platforms: [:ruby]
gem "puppet-module-posix-dev-r#{minor_version}", '~> 0.4', require: false, platforms: [:ruby]
gem "puppet-module-win-default-r#{minor_version}", '~> 0.4', require: false, platforms: [:mswin, :mingw, :x64_mingw]
gem "puppet-module-win-dev-r#{minor_version}", '~> 0.4', require: false, platforms: [:mswin, :mingw, :x64_mingw]
gem "puppet-debugger", '>= 0.18.0', require: false
gem "github_changelog_generator", require: false, git: 'https://github.com/skywinder/github-changelog-generator', ref: '20ee04ba1234e9e83eb2ffb5056e23d641c7a018' if Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.2.2')
end

puppet_version = ENV['PUPPET_GEM_VERSION']
Expand Down
5 changes: 5 additions & 0 deletions Puppetfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
forge 'https://forge.puppet.com'

# Modules from the Puppet Forge
mod 'puppetlabs-stdlib', '6.1.0'
mod 'WhatsARanjit-node_manager', '0.7.2'
36 changes: 29 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,16 +1,38 @@
# Puppet Enterprise Extra Large
# Puppet Enterprise (pe) Administration (adm) Module

This Puppet module contains Puppet Task Plans used to deploy an at-scale Puppet Enterprise architecture.
This Puppet module contains Bolt plans used to deploy and manage Puppet Enterprise infrastructure. Plans are provided to automate common lifecycle activities, in order to increase velocity and reduce the possibility of human error incurred by manually performing these activities.

Use this module to deploy Puppet Enterprise 2019.0.x Extra Large architecture.
The peadm module is able to deploy and manage Puppet Enterprise 2019.x Standard, Large, and Extra Large architectures.

* This deployment depends on and assumes the use of trusted facts. Specifically, `pp_role` and `pp_environment`.
* This deployment assumes that at least for PE infrastructure nodes, Puppet certnames are correct, resolvable FQDNs.
## Expectations

The peadm module is intended to be used only by Puppet Enterprise customers actively working with and being guided by Puppet Customer Success teams—specifically, the Professional Services and Solutions Architecture teams. Independent use is not recommended for production environments without a comprehensive understanding of the peadm module.

## Documentation

See this README file and any documents in the [documentation](documentation) directory.

## Architecture
Plans:

* [Provision](documentation/provision.md)
* [Upgrade](documentation/upgrade.md)
* [Convert](documentation/convert.md)
* [Status](documentation/status.md)

Reference:

* [PE Architecture Documentation](https://puppet.com/docs/pe/latest/choosing_an_architecture.html)
* [Classification](documentation/classification.md)
* [Architectures](documentation/architectures.md)
* [Testing](documentation/pre_post_checks.md)
* [Docker Based Examples](documentation/docker_examples.md)

## Requirements

Normally, if you are able to use peadm to set up an infrastructure and Puppet agent runs are all working, chances are you met all the requirements and don't have to worry. Sometimes Some notable requirements are highlighted below which may accidentally be adjusted by users, but which architectures deployed by this module rely on. These configuration requirements need to be maintained for the infrastructure to operate correctly.

* Classifier Data needs to be enabled. This feature is enabled by default on new installs, but can be disabled by users if they remove the relevant configuration from their global hiera.yaml file. See the [PE docs](https://puppet.com/docs/pe/latest/config_console.html#task-5039) for more information.

## Getting Help

![architecture](documentation/images/architecture.png)
To get help with issues concerning this module, please make use of [issues](https://github.com/puppetlabs/puppetlabs-peadm/issues) in the project on GitHub.
3 changes: 3 additions & 0 deletions Rakefile
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'puppet_litmus/rake_tasks' if Bundler.rubygems.find_name('puppet_litmus').any?
require 'puppetlabs_spec_helper/rake_tasks'
require 'puppet-syntax/tasks/puppet-syntax'
Expand Down Expand Up @@ -45,6 +47,7 @@ if Bundler.rubygems.find_name('github_changelog_generator').any?
raise "Set CHANGELOG_GITHUB_TOKEN environment variable eg 'export CHANGELOG_GITHUB_TOKEN=valid_token_here'" if Rake.application.top_level_tasks.include? "changelog" and ENV['CHANGELOG_GITHUB_TOKEN'].nil?
config.user = "#{changelog_user}"
config.project = "#{changelog_project}"
config.since_tag = "2.1.0"
config.future_release = "#{changelog_future_release}"
config.exclude_labels = ['maintenance']
config.header = "# Change log\n\nAll notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org)."
Expand Down
15 changes: 15 additions & 0 deletions documentation/architectures.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Architectures

The peadm module supports Puppet Enterprise 2019.x Standard, Large, and Extra Large architectures.

## Standard

Todo: diagram

## Large

Todo: diagram

## Extra Large

![architecture](images/architecture.png)
Loading