.fixtures.yml

@@ -12,4 +12,4 @@ fixtures:
       repo:           'https://github.com/puppetlabs/puppetlabs-selinux_core'
       puppet_version: '>= 6.0.0'
     stdlib:           'https://github.com/puppetlabs/puppetlabs-stdlib'
-    systemd:          'https://github.com/camptocamp/puppet-systemd'
+    systemd:          'https://github.com/voxpupuli/puppet-systemd'

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
       puppet_major_versions: ${{ steps.get_outputs.outputs.puppet_major_versions }}
       puppet_unit_test_matrix: ${{ steps.get_outputs.outputs.puppet_unit_test_matrix }}
     env:
-      BUNDLE_WITHOUT: development:release
+      BUNDLE_WITHOUT: development:system_tests:release
     steps:
       - uses: actions/checkout@v2
       - name: Setup ruby
@@ -65,8 +65,9 @@ jobs:
         setfile: ${{fromJson(needs.setup_matrix.outputs.beaker_setfiles)}}
         puppet: ${{fromJson(needs.setup_matrix.outputs.puppet_major_versions)}}
         pulpcore_version:
+          - '3.15'
           - '3.14'
-    name: Acceptance / ${{ matrix.puppet.name }} - ${{ matrix.setfile.name }}
+    name: Acceptance / ${{ matrix.puppet.name }} - ${{ matrix.setfile.name }} - Pulp ${{ matrix.pulpcore_version }}
     steps:
       - uses: actions/checkout@v2
       - name: Setup ruby

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## [5.1.0](https://github.com/theforeman/puppet-pulpcore/tree/5.1.0) (2021-10-29)
+
+[Full Changelog](https://github.com/theforeman/puppet-pulpcore/compare/5.0.0...5.1.0)
+
+**Implemented enhancements:**
+
+- Fixes [\#33766](https://projects.theforeman.org/issues/33766) - Support Pulpcore 3.15 [\#238](https://github.com/theforeman/puppet-pulpcore/pull/238) ([ekohl](https://github.com/ekohl))
+- Fixes [\#33765](https://projects.theforeman.org/issues/33765) - Use a system user without a login shell [\#237](https://github.com/theforeman/puppet-pulpcore/pull/237) ([ekohl](https://github.com/ekohl))
+- Refs [\#33751](https://projects.theforeman.org/issues/33751) - support ostree and python [\#236](https://github.com/theforeman/puppet-pulpcore/pull/236) ([jlsherrill](https://github.com/jlsherrill))
+- Fixes [\#33733](https://projects.theforeman.org/issues/33733) - generate key for db encryption [\#235](https://github.com/theforeman/puppet-pulpcore/pull/235) ([jlsherrill](https://github.com/jlsherrill))
+- use provides, not the python package name for plugins [\#233](https://github.com/theforeman/puppet-pulpcore/pull/233) ([evgeni](https://github.com/evgeni))
+- Fixes [\#33445](https://projects.theforeman.org/issues/33445): Increase default API gunicorn worker count [\#231](https://github.com/theforeman/puppet-pulpcore/pull/231) ([jlsherrill](https://github.com/jlsherrill))
+- Fixes [\#33446](https://projects.theforeman.org/issues/33446) - Allow configuring Pulpcore worker timeout [\#230](https://github.com/theforeman/puppet-pulpcore/pull/230) ([wbclark](https://github.com/wbclark))
+- Switch to puppet/systemd & allow puppet/redis 8 [\#228](https://github.com/theforeman/puppet-pulpcore/pull/228) ([ekohl](https://github.com/ekohl))
+- Fixes [\#33687](https://projects.theforeman.org/issues/33687) - Add warning not to directly edit settings.py [\#223](https://github.com/theforeman/puppet-pulpcore/pull/223) ([wbclark](https://github.com/wbclark))
+
+**Fixed bugs:**
+
+- Fixes [\#33744](https://projects.theforeman.org/issues/33744) - notify the socket service when the DB changes [\#234](https://github.com/theforeman/puppet-pulpcore/pull/234) ([evgeni](https://github.com/evgeni))
+
+**Closed issues:**
+
+- Ordering issue when adding repository [\#225](https://github.com/theforeman/puppet-pulpcore/issues/225)
+
 ## [5.0.0](https://github.com/theforeman/puppet-pulpcore/tree/5.0.0) (2021-07-27)
 
 [Full Changelog](https://github.com/theforeman/puppet-pulpcore/compare/4.0.1...5.0.0)

Gemfile

@@ -13,10 +13,11 @@ gem 'puppet-lint-param-docs', '>= 1.3.0', {"groups"=>["test"]}
 gem 'puppet-lint-spaceship_operator_without_tag-check', {"groups"=>["test"]}
 gem 'puppet-lint-strict_indent-check', {"groups"=>["test"]}
 gem 'puppet-lint-undef_in_function-check', {"groups"=>["test"]}
-gem 'voxpupuli-test', '~> 1.4'
+gem 'voxpupuli-test', '~> 1.4', {"groups"=>["test"]}
 gem 'github_changelog_generator', '>= 1.15.0', {"groups"=>["development"]}
 gem 'puppet_metadata', '~> 0.3'
 gem 'puppet-blacksmith', '>= 6.0.0', {"groups"=>["development"]}
 gem 'voxpupuli-acceptance', '~> 1.0', {"groups"=>["system_tests"]}
+gem 'puppetlabs_spec_helper', {"groups"=>["system_tests"]}
 
 # vim:ft=ruby

README.md

@@ -10,9 +10,15 @@ All supported versions are listed below. For every supported version, acceptance
 
 Supported operating systems are listed in `metadata.json` but individual releases can divert from that. For example, if Pulpcore x.y drops EL7, it will still be listed in metadata.json until all versions supported by the module have dropped it. Similarly, if x.z adds support for EL9, it'll be listed in `metadata.json` and all versions that don't support EL9 will have a note.
 
+### Pulpcore 3.15
+
+Default recommended version.
+
+Starting Pulpcore 3.15 the migration plugin is no longer built. Users should remove the plugin prior to upgrding. The [foreman_maintain Pulp 2 removal procedure](https://github.com/theforeman/foreman_maintain/blob/d49ece67f1dba761bb232229593765f61e01361a/definitions/procedures/pulp/remove.rb#L109-L160) is a good reference. Additionally the package `python3-pulp-2to3-migration` should be removed.
+
 ### Pulpcore 3.14
 
-Only supported version.
+At least pulpcore 3.14.8-2 (and matching plugins) should be used, as this version introduced virtual package names that are used in this module. Certguard 1.4.0-3 should be used to pull in the correct RHSM package.
 
 ## Installation layout
 
@@ -45,6 +51,22 @@ This results into the following structure, using `tree -pug`:
             └── [drwxr-x--- pulp     pulp    ]  tmp ($cache_dir)
 ```
 
+## Pulpcore settings
+
+The application will load settings from Django's defaults, Pulpcore's defaults, plus any overrides defined in the settings file at `${config_dir}/settings.py` (default: `/etc/pulp/settings.py`). The Django `diffsettings` tool is useful to check settings which are different from Django's defaults (i.e. all of the Pulpcore defaults which are not present in Django, plus any overrides defined in the settings file):
+
+```shell
+PULP_SETTINGS=/etc/pulp/settings.py pulpcore-manager diffsettings
+```
+
+For example, to check the current value of a Pulpcore setting such as `WORKER_TTL`:
+
+```shell
+PULP_SETTINGS=/etc/pulp/settings.py pulpcore-manager diffsettings | grep WORKER_TTL
+```
+
+This is useful for module parameter which configure Pulpcore settings but have an `undef` default, such as `$worker_ttl`.  When the param value is `undef`, the setting is omitted from `settings.py` and therefore Pulpcore's default is used.
+
 ## Service setup
 
 The module deploys a few systemd services:

Rakefile

@@ -1,10 +1,26 @@
 # This file is managed centrally by modulesync
 #   https://github.com/theforeman/foreman-installer-modulesync
 
-require 'voxpupuli/test/rake'
+# Attempt to load voxupuli-test (which pulls in puppetlabs_spec_helper),
+# otherwise attempt to load it directly.
+begin
+  require 'voxpupuli/test/rake'
+rescue LoadError
+  begin
+    require 'puppetlabs_spec_helper/rake_tasks'
+  rescue LoadError
+  end
+end
 
-# We use fixtures in our modules, which is not the default
-task :beaker => 'spec_prep'
+# load optional tasks for acceptance
+# only available if gem group releases is installed
+begin
+  require 'voxpupuli/acceptance/rake'
+rescue LoadError
+else
+  # We use fixtures in our modules, which is not the default
+  task :beaker => 'spec_prep'
+end
 
 # blacksmith isn't always present, e.g. on Travis with --without development
 begin

manifests/config.pp

@@ -1,7 +1,7 @@
 # Configures pulp3
 # @api private
 class pulpcore::config {
-  file { $pulpcore::config_dir:
+  file { [$pulpcore::config_dir, $pulpcore::certs_dir]:
     ensure => directory,
     owner  => 'root',
     group  => 'root',
@@ -51,4 +51,17 @@
     mode   => '0770',
   }
 
+  exec { 'Create database symmetric key':
+    path    => ['/bin', '/usr/bin'],
+    command => "openssl rand -base64 32 | tr '+/' '-_' > ${pulpcore::database_key_file}",
+    creates => $pulpcore::database_key_file,
+  }
+
+  file { $pulpcore::database_key_file:
+    owner   => 'root',
+    group   => $pulpcore::group,
+    mode    => '0640',
+    require => Exec['Create database symmetric key'],
+  }
+
 }

manifests/database.pp

@@ -13,6 +13,10 @@
       locale   => 'en_US.utf8',
       before   => Pulpcore::Admin['migrate --noinput'],
     }
+
+    # pulpcore-content fails to reconnect to the database, so schedule a restart whenever the db changes
+    # see https://pulp.plan.io/issues/9276 for details
+    Class['postgresql::server::service'] ~> Service['pulpcore-content.service']
   }
 
   pulpcore::admin { 'migrate --noinput':

manifests/init.pp

@@ -132,8 +132,15 @@
 #   available, likely results in performance degradation due to I/O blocking and is not recommended in most cases. Modifying this parameter should
 #   be done incrementally with benchmarking at each step to determine an optimal value for your deployment.
 #
+# @param worker_ttl
+#   Configure Pulpcore's WORKER_TTL setting. This is the number of seconds before a Pulpcore worker should be considered lost.
+#   If undefined, the setting will be omitted from the configuration file and Pulpcore will use its default value.
+#   To determine the current value in Pulpcore, see instructions about the diffsettings tool in this module's README.
+#   You should not need to modify this setting unless the application reports workers timing out while they are busy completing tasks.
+#   Modification should be performed incrementally to determine the least value that prevents false positive worker timeouts.
+#
 # @param use_rq_tasking_system
-#   Use the older RQ workers tasking system instead of the newer PostgreSQL tasking system introduced in Pulpcore 3.14.
+#   Use the older RQ workers tasking system instead of the newer PostgreSQL tasking system introduced in Pulpcore 3.14. This is deprecated in 3.15.
 #   Any benchmarking you did to optimize worker_count or other tasking related parameters will no longer be accurate after changing the tasking system.
 #   Do not modify this setting unless you understand the implications for performance and stability.
 #
@@ -211,11 +218,12 @@
   Pulpcore::ChecksumTypes $allowed_content_checksums = ['sha224', 'sha256', 'sha384', 'sha512'],
   String[1] $remote_user_environ_name = 'HTTP_REMOTE_USER',
   Integer[0] $worker_count = min(8, $facts['processors']['count']),
+  Optional[Integer[0]] $worker_ttl = undef,
   Boolean $use_rq_tasking_system = false,
   Boolean $service_enable = true,
   Boolean $service_ensure = true,
   Integer[0] $content_service_worker_count = (2*min(8, $facts['processors']['count']) + 1),
-  Integer[0] $api_service_worker_count = 1,
+  Integer[0] $api_service_worker_count = min(4, $facts['processors']['count']) + 1,
   Integer[0] $content_service_worker_timeout = 90,
   Integer[0] $api_service_worker_timeout = 90,
   Hash[String[1], String[1]] $api_client_auth_cn_map = {},
@@ -224,6 +232,8 @@
   Enum['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'] $log_level = 'INFO',
 ) {
   $settings_file = "${config_dir}/settings.py"
+  $certs_dir = "${config_dir}/certs"
+  $database_key_file = "${certs_dir}/database_fields.symmetric.key"
 
   contain pulpcore::install
   contain pulpcore::database

manifests/install.pp

@@ -3,7 +3,7 @@
 # @api private
 class pulpcore::install {
 
-  package { 'python3-pulpcore':
+  package { 'pulpcore':
     ensure => present,
   }
 
@@ -15,8 +15,10 @@
 
   user { $pulpcore::user:
     ensure     => present,
+    system     => true,
     gid        => $pulpcore::group,
     home       => $pulpcore::user_home,
+    shell      => '/sbin/nologin',
     managehome => false,
   }
 

manifests/plugin.pp

@@ -12,7 +12,7 @@
 # @param https_content
 #   Optional fragment for the Apache HTTPS vhost
 define pulpcore::plugin(
-  String $package_name = "python3-pulp-${title}",
+  String $package_name = "pulpcore-plugin(${title})",
   Optional[String] $config = undef,
   Optional[String] $http_content = undef,
   Optional[String] $https_content = undef,

manifests/plugin/certguard.pp

@@ -1,8 +1,5 @@
 # @summary Pulp Certguard plugin
 class pulpcore::plugin::certguard {
-  package { 'python3-subscription-manager-rhsm':
-    ensure => present,
-  }
-  -> pulpcore::plugin { 'certguard':
+  pulpcore::plugin { 'certguard':
   }
 }

manifests/plugin/migration.pp

@@ -43,7 +43,7 @@
   Stdlib::Absolutepath $mongo_db_ca_path = '/etc/pki/tls/certs/ca-bundle.crt',
 ) {
   pulpcore::plugin { 'migration':
-    package_name => 'python3-pulp-2to3-migration',
+    package_name => 'pulpcore-plugin(2to3-migration)',
     config       => template('pulpcore/migration-settings.py.erb'),
   }
 }

manifests/plugin/ostree.pp

@@ -0,0 +1,6 @@
+# @summary Pulp Ostree plugin
+#
+# This plugin is not packaged on EL7.
+class pulpcore::plugin::ostree {
+  pulpcore::plugin { 'ostree': }
+}

manifests/plugin/python.pp

@@ -0,0 +1,4 @@
+# @summary Pulp Python plugin
+class pulpcore::plugin::python {
+  pulpcore::plugin { 'python': }
+}

manifests/repo.pp

@@ -3,7 +3,7 @@
 # @param version
 #   The Pulpcore version to use
 class pulpcore::repo (
-  Pattern['^\d+\.\d+$'] $version = '3.14',
+  Pattern['^\d+\.\d+$'] $version = '3.15',
 ) {
   $context = {
     'version'   => $version,

metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "theforeman-pulpcore",
-  "version": "5.0.0",
+  "version": "5.1.0",
   "author": "theforeman",
   "summary": "Installs next generation Pulp server",
   "license": "GPL-3.0-or-later",
@@ -16,7 +16,7 @@
     },
     {
       "name": "puppet/redis",
-      "version_requirement": ">= 5.0.0 < 8.0.0"
+      "version_requirement": ">= 5.0.0 < 9.0.0"
     },
     {
       "name": "puppetlabs/apache",
@@ -27,7 +27,7 @@
       "version_requirement": ">= 6.5.0 < 8.0.0"
     },
     {
-      "name": "camptocamp/systemd",
+      "name": "puppet/systemd",
       "version_requirement": ">= 2.2.0 < 4.0.0"
     },
     {

spec/acceptance/basic_spec.rb

@@ -79,7 +79,7 @@ class { 'pulpcore':
     its(:exit_status) { is_expected.to eq 0 }
   end
 
-  describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py rq info -c pulpcore.rqconfig") do
+  describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py /usr/libexec/pulpcore/rq info -c pulpcore.rqconfig") do
     its(:stdout) { is_expected.to match(/^0 workers, /) }
     its(:stdout) { is_expected.not_to match(/^resource-manager /) }
     its(:exit_status) { is_expected.to eq 0 }

spec/acceptance/disable_new_tasking_system_spec.rb

@@ -62,7 +62,7 @@ class { 'pulpcore':
       its(:exit_status) { is_expected.to eq 0 }
     end
 
-    describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py rq info -c pulpcore.rqconfig") do
+    describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py /usr/libexec/pulpcore/rq info -c pulpcore.rqconfig") do
       its(:stdout) { is_expected.to match(/^0 workers, /) }
       its(:stdout) { is_expected.not_to match(/^resource-manager /) }
       its(:exit_status) { is_expected.to eq 0 }
@@ -128,7 +128,7 @@ class { 'pulpcore':
       its(:exit_status) { is_expected.to eq 0 }
     end
 
-    describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py rq info -c pulpcore.rqconfig") do
+    describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py /usr/libexec/pulpcore/rq info -c pulpcore.rqconfig") do
       its(:stdout) { is_expected.to match(/^2 workers, /) }
       its(:stdout) { is_expected.to match(/^resource-manager /) }
       its(:exit_status) { is_expected.to eq 0 }

spec/acceptance/enable_new_tasking_system_spec.rb

@@ -62,7 +62,7 @@ class { 'pulpcore':
       its(:exit_status) { is_expected.to eq 0 }
     end
 
-    describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py rq info -c pulpcore.rqconfig") do
+    describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py /usr/libexec/pulpcore/rq info -c pulpcore.rqconfig") do
       its(:stdout) { is_expected.to match(/^2 workers, /) }
       its(:stdout) { is_expected.to match(/^resource-manager /) }
       its(:exit_status) { is_expected.to eq 0 }
@@ -128,7 +128,7 @@ class { 'pulpcore':
       its(:exit_status) { is_expected.to eq 0 }
     end
 
-    describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py rq info -c pulpcore.rqconfig") do
+    describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py /usr/libexec/pulpcore/rq info -c pulpcore.rqconfig") do
       its(:stdout) { is_expected.to match(/^0 workers, /) }
       its(:stdout) { is_expected.not_to match(/^resource-manager /) }
       its(:exit_status) { is_expected.to eq 0 }

spec/acceptance/plugins_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper_acceptance'
 
-describe 'basic installation' do
+describe 'Installation with all plugins' do
   it_behaves_like 'an idempotent resource' do
     let(:manifest) do
       <<-PUPPET
@@ -10,7 +10,15 @@
       include pulpcore::plugin::container
       include pulpcore::plugin::deb
       include pulpcore::plugin::file
-      include pulpcore::plugin::migration
+      if fact('pulpcore_version') == '3.14' {
+        include pulpcore::plugin::migration
+      }
+      if versioncmp(fact('pulpcore_version'), '3.15') >= 0 {
+        if versioncmp(fact('os.release.major'), '8') >= 0 {
+          include pulpcore::plugin::ostree
+        }
+        include pulpcore::plugin::python
+      }
       include pulpcore::plugin::rpm
       PUPPET
     end