Showing with 230 additions and 315 deletions.
  1. +3 −0 .gitignore
  2. +13 −14 .gitlab-ci.yml
  3. +5 −0 .pdkignore
  4. +19 −8 .travis.yml
  5. +1 −1 Rakefile
  6. BIN documentation/images/pe-xl-classification.png
  7. +82 −125 manifests/setup/node_manager.pp
  8. +4 −4 metadata.json
  9. +0 −2 plans/configure.pp
  10. +0 −2 plans/init.pp
  11. +1 −1 spec/spec_helper.rb
  12. +10 −14 tasks/configure_node_groups.json
  13. +92 −144 tasks/configure_node_groups.sh
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,6 @@
/convert_report.txt
/update_report.txt
.DS_Store
.project
.envrc
/inventory.yaml
27 changes: 13 additions & 14 deletions .gitlab-ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,28 +15,27 @@ before_script:
- bundle -v
- bundle install --without system_tests --path vendor/bundle --jobs $(nproc)

parallel_spec-Ruby 2.1.9-Puppet ~> 4.0:
stage: unit
image: ruby:2.1.9
syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop-Ruby 2.5.3-Puppet ~> 6:
stage: syntax
image: ruby:2.5.3
script:
- bundle exec rake parallel_spec
- bundle exec rake syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop
variables:
PUPPET_GEM_VERSION: '~> 4.0'
RUBYGEMS_VERSION: '2.7.8'
PUPPET_GEM_VERSION: '~> 6'

syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop-Ruby 2.4.4-Puppet ~> 5.5:
stage: syntax
image: ruby:2.4.4
parallel_spec-Ruby 2.5.3-Puppet ~> 6:
stage: unit
image: ruby:2.5.3
script:
- bundle exec rake syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop
- bundle exec rake parallel_spec
variables:
PUPPET_GEM_VERSION: '~> 5.5'
PUPPET_GEM_VERSION: '~> 6'

parallel_spec-Ruby 2.4.4-Puppet ~> 5.5:
parallel_spec-Ruby 2.4.5-Puppet ~> 5:
stage: unit
image: ruby:2.4.4
image: ruby:2.4.5
script:
- bundle exec rake parallel_spec
variables:
PUPPET_GEM_VERSION: '~> 5.5'
PUPPET_GEM_VERSION: '~> 5'

5 changes: 5 additions & 0 deletions .pdkignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
/convert_report.txt
/update_report.txt
.DS_Store
.project
.envrc
/inventory.yaml
/appveyor.yml
/.fixtures.yml
/Gemfile
Expand All @@ -30,8 +33,10 @@
/.gitlab-ci.yml
/.pdkignore
/Rakefile
/rakelib/
/.rspec
/.rubocop.yml
/.travis.yml
/.yardopts
/spec/
/.vscode/
27 changes: 19 additions & 8 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,20 +12,31 @@ script:
- 'bundle exec rake $CHECK'
bundler_args: --without system_tests
rvm:
- 2.5.1
env:
global:
- BEAKER_PUPPET_COLLECTION=puppet6 PUPPET_GEM_VERSION="~> 6.0"
- 2.5.3
stages:
- static
- spec
- acceptance
-
if: tag =~ ^v\d
name: deploy
matrix:
fast_finish: true
include:
-
env: CHECK="syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop"
-
env: CHECK=parallel_spec
env: CHECK="check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop syntax lint metadata_lint"
stage: static
-
env: PUPPET_GEM_VERSION="~> 5.0" CHECK=parallel_spec
rvm: 2.4.4
rvm: 2.4.5
stage: spec
-
env: PUPPET_GEM_VERSION="~> 6.0" CHECK=parallel_spec
rvm: 2.5.3
stage: spec
-
env: DEPLOY_TO_FORGE=yes
stage: deploy
branches:
only:
- master
Expand Down
2 changes: 1 addition & 1 deletion Rakefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ end

def changelog_future_release
return unless Rake.application.top_level_tasks.include? "changelog"
returnVal = JSON.load(File.read('metadata.json'))['version']
returnVal = "v%s" % JSON.load(File.read('metadata.json'))['version']
raise "unable to find the future_release (version) in metadata.json" if returnVal.nil?
puts "GitHubChangelogGenerator future_release:#{returnVal}"
returnVal
Expand Down
Binary file modified documentation/images/pe-xl-classification.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
207 changes: 82 additions & 125 deletions manifests/setup/node_manager.pp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,20 +12,34 @@
# }'
#
class pe_xl::setup::node_manager (
String[1] $master_host,
String[1] $master_replica_host,
String[1] $puppetdb_database_host,
String[1] $puppetdb_database_replica_host,
String[1] $compiler_pool_address,
Boolean $manage_environment_groups = true,
Pattern[/\A[a-z0-9_]+\Z/] $default_environment = 'production',
Array[Pattern[/\A[a-z0-9_]+\Z/]] $environments = ['production'],
String[1] $master_host,
String[1] $puppetdb_database_host,
String[1] $compiler_pool_address,

Optional[String[1]] $master_replica_host = undef,
Optional[String[1]] $puppetdb_database_replica_host = undef,
) {

if ([$master_replica_host, $puppetdb_database_replica_host].filter |$_| { $_ }.size == 1) {
fail('Must pass both master_replica_host and puppetdb_database_replica_host, or neither')
}

##################################################
# PE INFRASTRUCTURE GROUPS
##################################################

# Hiera data tuning for compilers
$compiler_data = {
'puppet_enterprise::profile::puppetdb' => {
'gc_interval' => '0',
},
'puppet_enterprise::puppetdb' => {
'command_processing_threads' => 2,
'write_maximum_pool_size' => 4,
'read_maximum_pool_size' => 8,
},
}

# We modify this group's rule such that all PE infrastructure nodes will be
# members.
node_group { 'PE Infrastructure Agent':
Expand All @@ -46,17 +60,18 @@
},
}

# We need to pre-create this group so that the master replica can be
# identified as running PuppetDB, so that Puppet will create a pg_ident
# authorization rule for it on the PostgreSQL nodes.
node_group { 'PE HA Replica':
ensure => 'present',
parent => 'PE Infrastructure',
rule => ['or', ['=', 'name', $master_replica_host]],
classes => {
'puppet_enterprise::profile::primary_master_replica' => { }
# This class has to be included here because puppet_enterprise is declared
# in the console with parameters. It is therefore not possible to include
# puppet_enterprise::profile::database in code without causing a conflict.
node_group { 'PE Database':
ensure => present,
parent => 'PE Infrastructure',
environment => 'production',
override_environment => false,
rule => ['and', ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::puppetdb_database']],
classes => {
'puppet_enterprise::profile::database' => { },
},
variables => { 'pe_xl_replica' => true },
}

# Create data-only groups to store PuppetDB PostgreSQL database configuration
Expand All @@ -78,39 +93,8 @@
},
}

node_group { 'PE Master B':
ensure => present,
parent => 'PE Infrastructure',
rule => ['and',
['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::master'],
['=', ['trusted', 'extensions', 'pp_cluster'], 'B'],
],
data => {
'puppet_enterprise::profile::primary_master_replica' => {
'database_host_puppetdb' => $puppetdb_database_replica_host,
},
'puppet_enterprise::profile::puppetdb' => {
'database_host' => $puppetdb_database_replica_host,
},
},
}

# Hiera data tuning for compilers
$compiler_data = {
'puppet_enterprise::profile::puppetdb' => {
'gc_interval' => '0',
},
'puppet_enterprise::puppetdb' => {
'command_processing_threads' => 2,
'write_maximum_pool_size' => 4,
'read_maximum_pool_size' => 10,
},
}

# Configure the compilers for HA, grouped into two pools, each pool
# having an affinity for one "availability zone" or the other. Even with an
# affinity, note that data from each compiler is replicated to both
# "availability zones".
# Configure the A pool for compilers. There are up to two pools for HA, each
# having an affinity for one "availability zone" or the other.
node_group { 'PE Compiler Group A':
ensure => 'present',
parent => 'PE Master',
Expand All @@ -123,91 +107,64 @@
'database_host' => $puppetdb_database_host,
},
'puppet_enterprise::profile::master' => {
'puppetdb_host' => ['${clientcert}', $master_replica_host], # lint:ignore:single_quote_string_with_variables
'puppetdb_host' => ['${clientcert}', $master_replica_host].filter |$_| { $_ }, # lint:ignore:single_quote_string_with_variables
'puppetdb_port' => [8081],
}
},
data => $compiler_data,
}

node_group { 'PE Compiler Group B':
ensure => 'present',
parent => 'PE Master',
rule => ['and',
['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::compiler'],
['=', ['trusted', 'extensions', 'pp_cluster'], 'B'],
],
classes => {
'puppet_enterprise::profile::puppetdb' => {
'database_host' => $puppetdb_database_replica_host,
# Create the replica and B groups if a replica master and database host are
# supplied
if ($master_replica_host and $puppetdb_database_replica_host) {
# We need to pre-create this group so that the master replica can be
# identified as running PuppetDB, so that Puppet will create a pg_ident
# authorization rule for it on the PostgreSQL nodes.
node_group { 'PE HA Replica':
ensure => 'present',
parent => 'PE Infrastructure',
rule => ['or', ['=', 'name', $master_replica_host]],
classes => {
'puppet_enterprise::profile::primary_master_replica' => { }
},
'puppet_enterprise::profile::master' => {
'puppetdb_host' => ['${clientcert}', $master_host], # lint:ignore:single_quote_string_with_variables
'puppetdb_port' => [8081],
}
},
data => $compiler_data,
}

# This class has to be included here because puppet_enterprise is declared
# in the console with parameters. It is therefore not possible to include
# puppet_enterprise::profile::database in code without causing a conflict.
node_group { 'PE Database':
ensure => present,
parent => 'PE Infrastructure',
environment => 'production',
override_environment => false,
rule => ['and', ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::puppetdb_database']],
classes => {
'puppet_enterprise::profile::database' => { },
},
}


if ($manage_environment_groups) {

##################################################
# ENVIRONMENT GROUPS
##################################################

node_group { 'All Environments':
ensure => present,
description => 'Environment group parent and default',
environment => $default_environment,
override_environment => true,
parent => 'All Nodes',
rule => ['and', ['~', 'name', '.*']],
variables => { 'pe_xl_replica' => true },
}

node_group { 'Agent-specified environment':
ensure => present,
description => 'This environment group exists for unusual testing and development only. Expect it to be empty',
environment => 'agent-specified',
override_environment => true,
parent => 'All Environments',
rule => [ ],
node_group { 'PE Master B':
ensure => present,
parent => 'PE Infrastructure',
rule => ['and',
['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::master'],
['=', ['trusted', 'extensions', 'pp_cluster'], 'B'],
],
data => {
'puppet_enterprise::profile::primary_master_replica' => {
'database_host_puppetdb' => $puppetdb_database_replica_host,
},
'puppet_enterprise::profile::puppetdb' => {
'database_host' => $puppetdb_database_replica_host,
},
},
}

$environments.each |$env| {
$title_env = capitalize($env)

node_group { "${title_env} environment":
ensure => present,
environment => $env,
override_environment => true,
parent => 'All Environments',
rule => ['and', ['=', ['trusted', 'extensions', 'pp_environment'], $env]],
}

node_group { "${title_env} one-time run exception":
ensure => present,
description => "Allow ${env} nodes to request a different puppet environment for a one-time run",
environment => 'agent-specified',
override_environment => true,
parent => "${title_env} environment",
rule => ['and', ['~', ['fact', 'agent_specified_environment'], '.+']],
}
node_group { 'PE Compiler Group B':
ensure => 'present',
parent => 'PE Master',
rule => ['and',
['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::compiler'],
['=', ['trusted', 'extensions', 'pp_cluster'], 'B'],
],
classes => {
'puppet_enterprise::profile::puppetdb' => {
'database_host' => $puppetdb_database_replica_host,
},
'puppet_enterprise::profile::master' => {
'puppetdb_host' => ['${clientcert}', $master_host], # lint:ignore:single_quote_string_with_variables
'puppetdb_port' => [8081],
}
},
data => $compiler_data,
}

}

}
8 changes: 4 additions & 4 deletions metadata.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "reidmv-pe_xl",
"version": "0.2.2",
"version": "0.2.3",
"author": "Reid Vandewiele",
"summary": "Profile classes used to deploy an at-scale Puppet Enterprise architecture",
"license": "Apache-2.0",
Expand Down Expand Up @@ -49,7 +49,7 @@
"version_requirement": ">= 6.0.0 < 7.0.0"
}
],
"pdk-version": "1.9.0",
"template-url": "https://github.com/puppetlabs/pdk-templates.git",
"template-ref": "1.9.0-0-g7281db5"
"pdk-version": "1.10.0",
"template-url": "https://github.com/puppetlabs/pdk-templates.git#1.10.0",
"template-ref": "1.10.0-0-gbba9ac3"
}
Loading