Merge pull request #123 from m0dular/SUP-3972-activity_tables

(SUP-3972) Add activity tables to repack

Author: MartyEwings

data/common.yaml

@@ -0,0 +1,24 @@
+pe_databases::facts_tables_repack_timer:
+  'Tue,Sat *-*-* 04:30:00'
+pe_databases::catalogs_tables_repack_timer:
+  'Sun,Thu *-*-* 04:30:00'
+pe_databases::other_tables_repack_timer:
+  '*-*-20 05:30:00'
+pe_databases::activity_tables_repack_timer:
+  'Wed,Fri *-*-* 04:30:00'
+pe_databases::pg_repack::fact_tables:
+  - factsets
+  - fact_paths
+pe_databases::pg_repack::catalog_tables:
+  - catalogs
+  - catalog_resources
+  - catalog_inputs
+  - edges
+  - certnames
+pe_databases::pg_repack::other_tables:
+  - producers
+  - resource_params
+  - resource_params_cache
+pe_databases::pg_repack::activity_tables:
+  - events
+  - event_commits

hiera.yaml

@@ -0,0 +1,10 @@
+---
+version: 5
+
+defaults:  # Used for any hierarchy level that omits these keys.
+  datadir: data         # This path is relative to hiera.yaml's directory.
+  data_hash: yaml_data  # Use the built-in YAML backend.
+
+hierarchy:
+  - name: 'common'
+    path: 'common.yaml'

manifests/collect.pp

@@ -3,12 +3,13 @@
 # @param command [String] defined in pg_repack.pp
 # @param disable_maintenance [Boolean] to disable maintenance mode (Default: false)
 # @param on_cal [String] values can be found in pg_repack.pp
-#
+# @param tables [Array] Array of tables to repack
 define pe_databases::collect (
   String  $database_type       = $title,
   String  $command             = undef,
   Boolean $disable_maintenance = false,
   String  $on_cal              = undef,
+  Array   $tables              = undef,
 ) {
   Service {
     notify  => Exec['pe_databases_daemon_reload'],
@@ -27,9 +28,13 @@
     default => present
   }
 
+  $tables_command = $tables.map |$table| { "-t ${table}" }.join(' ')
+
+  $repack_command = "${command} ${tables_command}"
+
   file { "/etc/systemd/system/pe_databases-${database_type}.service":
     ensure  => $ensure_file,
-    content => epp('pe_databases/service.epp', { 'tables' => $database_type, 'command' => $command }),
+    content => epp('pe_databases/service.epp', { 'tables' => $database_type, 'command' => $repack_command }),
   }
   file { "/etc/systemd/system/pe_databases-${database_type}.timer":
     ensure  => $ensure_file,

manifests/init.pp

@@ -9,20 +9,23 @@
 # @param facts_tables_repack_timer [String] The Systemd timer for the pg_repack job affecting the 'facts' tables
 # @param catalogs_tables_repack_timer [String]The Systemd timer for the pg_repack job affecting the 'catalog' tables
 # @param other_tables_repack_timer [String] The Systemd timer for the pg_repack job affecting the 'other' tables
+# @param activity_tables_repack_timer [String] The Systemd timer for the pg_repack job affecting the 'activity' tables
 # @param manage_postgresql_settings [Boolean] Deprecated Parameter will be removed in future releases
 # @param manage_table_settings [Boolean] Deprecated Parameter will be removed in future releases
 # @param reports_tables_repack_timer [String] Deprecated Parameter will be removed in future releases
 # @param resource_events_tables_repack_timer [String] Deprecated Parameter will be removed in future releases
 class pe_databases (
+  # Provided by module data
+  String[1] $facts_tables_repack_timer,
+  String[1] $catalogs_tables_repack_timer,
+  String[1] $other_tables_repack_timer,
+  String[1] $activity_tables_repack_timer,
   Boolean $manage_database_maintenance           = true,
   Boolean $disable_maintenance                   = false,
   Optional[Boolean] $manage_postgresql_settings  = undef,
   Optional[Boolean] $manage_table_settings       = undef,
   String[1] $install_dir                         = '/opt/puppetlabs/pe_databases',
   String[1] $scripts_dir                         = "${install_dir}/scripts",
-  String[1] $facts_tables_repack_timer           = 'Tue,Sat *-*-* 04:30:00',
-  String[1] $catalogs_tables_repack_timer        = 'Sun,Thu *-*-* 04:30:00',
-  String[1] $other_tables_repack_timer           = '*-*-20 05:30:00',
   Optional[String] $reports_tables_repack_timer         = undef,
   Optional[String] $resource_events_tables_repack_timer = undef,
 ) {

manifests/pg_repack.pp

@@ -1,23 +1,33 @@
 # Maintenance pg_repack
 #
-# @summary 
-#   Provides systemd timers to pg_repack tables in the pe-puppetdb database
-# 
+# @summary
+#   Provides systemd timers to pg_repack tables in a given database
+# @param fact_tables [Array] Array of 'fact' tables to repack
+# @param catalog_tables [Array] Array of 'catalog' tables to repack
+# @param other_tables [Array] Array of 'other' tables to repack
+# @param activity_tables [Array] Array of 'activity' tables to repack
 # @param disable_maintenance [Boolean] true or false (Default: false)
 #   Disable or enable maintenance mode
 # @param jobs [Integer] How many jobs to run in parallel
 # @param facts_tables_repack_timer [String] The Systemd timer for the pg_repack job affecting the 'facts' tables
 # @param catalogs_tables_repack_timer [String]The Systemd timer for the pg_repack job affecting the 'catalog' tables
 # @param other_tables_repack_timer [String] The Systemd timer for the pg_repack job affecting the 'other' tables
+# @param activity_tables_repack_timer [String] The Systemd timer for the pg_repack job affecting the 'activity' tables
 # @param reports_tables_repack_timer [String] Deprecated Parameter will be removed in future releases
 # @param resource_events_tables_repack_timer [String] Deprecated Parameter will be removed in future releases
 class pe_databases::pg_repack (
-  Boolean $disable_maintenance                          = false,
-  Integer $jobs                                         = $facts['processors']['count'] / 4,
-  String[1] $facts_tables_repack_timer                  = $pe_databases::facts_tables_repack_timer,
-  String[1] $catalogs_tables_repack_timer               = $pe_databases::catalogs_tables_repack_timer,
-  String[1] $other_tables_repack_timer                  = $pe_databases::other_tables_repack_timer,
-  Optional[String] $reports_tables_repack_timer         = undef,
+  # Provided by module data
+  Array $fact_tables,
+  Array $catalog_tables,
+  Array $other_tables,
+  Array $activity_tables,
+  Boolean $disable_maintenance = false,
+  Integer $jobs = $facts['processors']['count'] / 4,
+  String[1] $facts_tables_repack_timer = $pe_databases::facts_tables_repack_timer,
+  String[1] $catalogs_tables_repack_timer = $pe_databases::catalogs_tables_repack_timer,
+  String[1] $other_tables_repack_timer = $pe_databases::other_tables_repack_timer,
+  String[1] $activity_tables_repack_timer = $pe_databases::activity_tables_repack_timer,
+  Optional[String] $reports_tables_repack_timer = undef,
   Optional[String] $resource_events_tables_repack_timer = undef,
 ) {
   puppet_enterprise::deprecated_parameter { 'pe_databases::pg_repack::reports_tables_repack_timer': }
@@ -26,28 +36,34 @@
   $postgresql_version = $facts['pe_postgresql_info']['installed_server_version']
   $repack_executable = "/opt/puppetlabs/server/apps/postgresql/${postgresql_version}/bin/pg_repack"
 
-  $repack_cmd = "${repack_executable} -d pe-puppetdb --jobs ${jobs}"
-
-  $fact_tables = '-t factsets -t fact_paths'
-  $catalog_tables = '-t catalogs -t catalog_resources -t catalog_inputs -t edges -t certnames'
-  $other_tables = '-t producers -t resource_params -t resource_params_cache'
+  $repack_cmd = "${repack_executable} --jobs ${jobs}"
 
   pe_databases::collect { 'facts':
     disable_maintenance => $disable_maintenance,
-    command             => "${repack_cmd} ${fact_tables}",
+    command             => "${repack_cmd} -d pe-puppetdb",
     on_cal              => $facts_tables_repack_timer,
+    tables              => $fact_tables,
   }
 
   pe_databases::collect { 'catalogs':
     disable_maintenance => $disable_maintenance,
-    command             => "${repack_cmd} ${catalog_tables}",
+    command             => "${repack_cmd} -d pe-puppetdb",
     on_cal              => $catalogs_tables_repack_timer,
+    tables              => $catalog_tables,
   }
 
   pe_databases::collect { 'other':
     disable_maintenance => $disable_maintenance,
-    command             => "${repack_cmd} ${other_tables}",
+    command             => "${repack_cmd} -d pe-puppetdb",
     on_cal              => $other_tables_repack_timer,
+    tables              => $other_tables,
+  }
+
+  pe_databases::collect { 'activity':
+    disable_maintenance => $disable_maintenance,
+    command             => "${repack_cmd} -d pe-activity",
+    on_cal              => $activity_tables_repack_timer,
+    tables              => $activity_tables,
   }
 
   # Ensure legacy vaccum and pg_repack crons are purged.

spec/classes/pg_repack_spec.rb

@@ -2,20 +2,28 @@
 
 describe 'pe_databases::pg_repack' do
   let(:facts) { { processors: { count: 4 } } }
-  let(:repack_cmd) { '/opt/puppetlabs/server/apps/postgresql/11/bin/pg_repack -d pe-puppetdb --jobs 1' }
+  let(:repack_cmd) { '/opt/puppetlabs/server/apps/postgresql/11/bin/pg_repack --jobs 1' }
   let(:tables_hash) do
     {
       facts: {
         tables: '-t factsets -t fact_paths',
-        schedule: 'Tue,Sat \*-\*-\* 04:30:00'
+        schedule: 'Tue,Sat \*-\*-\* 04:30:00',
+        database: '-d pe-puppetdb',
       },
       catalogs: {
         tables: '-t catalogs -t catalog_resources -t catalog_inputs -t edges -t certnames',
         schedule: 'Sun,Thu \*-\*-\* 04:30:00',
+        database: '-d pe-puppetdb',
       },
       other: {
         tables: '-t producers -t resource_params -t resource_params_cache',
         schedule: '\*-\*-20 05:30:00',
+        database: '-d pe-puppetdb',
+      },
+      activity: {
+        tables: '-t events -t event_commits',
+        schedule: 'Wed,Fri \*-\*-\* 04:30:00',
+        database: '-d pe-activity',
       }
     }
   end
@@ -48,7 +56,7 @@
       tables_hash.each do |name, val|
         is_expected.to contain_pe_databases__collect(name).with(
           disable_maintenance: false,
-          command: "#{repack_cmd} #{val[:tables]}",
+          command: "#{repack_cmd} #{val[:database]}",
           # Strip the backslash character because this is not a regex
           on_cal: (val[:schedule]).to_s.tr('\\', ''),
         )
@@ -58,7 +66,7 @@
 
         is_expected.to contain_file("/etc/systemd/system/pe_databases-#{name}.timer").with_content(%r{OnCalendar=#{val[:schedule]}})
         is_expected.to contain_file("/etc/systemd/system/pe_databases-#{name}.service").with_content(
-          %r{ExecStart=#{repack_cmd} #{val[:tables]}},
+          %r{ExecStart=#{repack_cmd} #{val[:database]} #{val[:tables]}},
         )
 
         [

spec/defines/collect_spec.rb

@@ -13,6 +13,7 @@
       {
         command: 'foo',
         on_cal: 'bar',
+        tables: ['baz'],
       }
     end
 
@@ -22,7 +23,7 @@
 
       is_expected.to contain_file('/etc/systemd/system/pe_databases-test.timer').with_content(%r{bar})
       is_expected.to contain_file('/etc/systemd/system/pe_databases-test.service').with_content(
-        %r{ExecStart=foo},
+        %r{ExecStart=foo.*-t baz},
       )
 
       is_expected.to contain_service('pe_databases-test.service').that_notifies(
@@ -47,6 +48,7 @@
         disable_maintenance: true,
         command: 'foo',
         on_cal: 'bar',
+        tables: ['baz'],
       }
     end