chore(unit-tests) generate test fixtures from Puppetfile

.gitignore

@@ -13,3 +13,4 @@ log/
 tmp/
 vendor/
 *.log
+.fixtures.yml

Gemfile

@@ -15,6 +15,7 @@ gem 'puppetlabs_spec_helper'
 gem 'pry'
 gem 'serverspec'
 gem 'hiera-eyaml', '~>3.2.2'
+gem 'generate-puppetfile'
 
 group :development do
   gem 'debugger', :platform => :mri_19

Gemfile.lock

@@ -46,6 +46,7 @@ GEM
     byebug (8.2.1)
     coderay (1.1.0)
     colored (1.2)
+    colorize (0.8.1)
     columnize (0.9.0)
     cri (2.6.1)
       colored (~> 1.2)
@@ -168,6 +169,8 @@ GEM
       fog-core
       nokogiri (>= 1.5.11, < 2.0.0)
     formatador (0.2.5)
+    generate-puppetfile (1.1.0)
+      colorize
     gettext (3.2.2)
       locale (>= 2.0.5)
       text (>= 1.3.0)
@@ -339,6 +342,7 @@ DEPENDENCIES
   byebug
   debugger
   debugger-pry
+  generate-puppetfile
   hiera-eyaml (~> 3.2.2)
   parallel_tests
   pry

Jenkinsfile

@@ -1,54 +1,39 @@
 #!/usr/bin/env groovy
 
 pipeline {
-    agent none
+    agent { label 'ruby' }
 
     options {
         buildDiscarder(logRotator(numToKeepStr: '10'))
-        timeout(time: 30, unit: 'MINUTES')
+        timeout(time: 15, unit: 'MINUTES')
         timestamps()
     }
 
-    environment {
-        LANG = 'en_US.UTF-8'
-        LC_CTYPE = 'en_US.UTF-8'
-        /* These environment variables make it feasible for Git to clone properly while
-         * inside the wacky confines of a Docker container
-         */
-        GIT_COMMITTER_EMAIL = 'git@example.com'
-        GIT_COMMITTER_NAME = 'Git'
-        GIT_AUTHOR_NAME = 'Git'
-        GIT_AUTHOR_EMAIL = 'git@example.com'
-    }
-
     stages {
+        stage('Prepare Puppet Project') {
+            steps {
+                // Install Dependencies once for all
+                sh 'bash ./ci/00_setupgems.sh'
+                // For auditing purposes: if test are failing with "module not found" or "object not found" for instance
+                archiveArtifacts '.fixtures.yml'
+            }
+        }
         stage('Verify') {
             failFast true
             parallel {
                 stage('Syntax') {
-                    agent { label 'ruby' }
                     steps {
-                        sh 'HOME=$PWD bundle install --without development plugins --path vendor/gems'
-                        sh 'HOME=$PWD bundle exec rake spec_clean spec_prep'
                         sh 'bundle exec rake lint'
                     }
                 }
                 stage('Profiles') {
-                    agent { label 'ruby' }
                     steps {
-                        sh 'HOME=$PWD bundle install --without development plugins --path vendor/gems'
-                        sh 'HOME=$PWD bundle exec rake spec_clean spec_prep'
                         sh 'bundle exec parallel_rspec spec/classes/profile'
-                        junit 'tmp/rspec*.xml'
                     }
                 }
                 stage('Roles') {
-                    agent { label 'ruby' }
                     steps {
-                        sh 'HOME=$PWD bundle install --without development plugins --path vendor/gems'
-                        sh 'HOME=$PWD bundle exec rake spec_clean spec_prep'
                         sh 'bundle exec parallel_rspec spec/classes/role'
-                        junit 'tmp/rspec*.xml'
                     }
                 }
             }

Puppetfile

@@ -33,27 +33,8 @@ mod 'puppetlabs/docker', '4.4.0'
 # Deps for docker
 mod 'puppetlabs/apt', '8.4.1'
 
-# Dependencies for the Puppet IRC report processor, using our forked version
-# which updates on any changed status
-mod 'irc', :git => 'https://github.com/jenkins-infra/puppet-irc.git',
-           :ref => '4e5e437'
-
-# Needed for managing our accounts in hiera, this fork contains the pull
-# request which adds support for multiple SSH keys:
-# <https://github.com/torrancew/puppet-account/pull/18>
-mod 'account', :git => 'https://github.com/jenkins-infra/puppet-account.git',
-               :ref => '1deebe9'
-
-mod 'jenkins_keys',
-  :git => 'git@github.com:jenkins-infra/jenkins-keys.git',
-  :ref => 'eeb7db7'
-
 # Apache and its dependencies
 mod "puppetlabs/apache", '3.5.0'
-# Used internally to gzip compress rotated logs
-mod 'apachelogcompressor',
-        :git => 'https://github.com/jenkins-infra/puppet-apachelogcompressor.git',
-        :ref => '0113d7b'
 
 mod "puppetlabs/concat", '5.2.0'
 
@@ -87,3 +68,29 @@ mod 'erwbgy/limits', '0.3.1'
 
 # For managing sysctl configuration
 mod 'herculesteam-augeasproviders_sysctl', '2.2.1'
+
+##### The following custom puppet modules must be specified with 1 attribute per line
+# Example:
+#  mod 'modulename'
+#    :git => <git URL>,
+#    :ref => <git ref>,
+#####
+
+# Dependencies for the Puppet IRC report processor, using our forked version
+# which updates on any changed status
+mod 'irc',
+  :git => 'https://github.com/jenkins-infra/puppet-irc.git',
+  :ref => '4e5e437'
+# Needed for managing our accounts in hiera, this fork contains the pull
+# request which adds support for multiple SSH keys:
+# <https://github.com/torrancew/puppet-account/pull/18>
+mod 'account',
+  :git => 'https://github.com/jenkins-infra/puppet-account.git',
+  :ref => '1deebe9'
+mod 'jenkins_keys',
+  :git => 'git@github.com:jenkins-infra/jenkins-keys.git',
+  :ref => 'eeb7db7'
+# Used internally to gzip compress rotated logs
+mod 'apachelogcompressor',
+  :git => 'https://github.com/jenkins-infra/puppet-apachelogcompressor.git',
+  :ref => '0113d7b'

README.md

@@ -1,10 +1,6 @@
 # Jenkins Infra
 
-[![Build Status](https://ci.jenkins.io/buildStatus/icon?job=Infra/jenkins-infra/production)](https://ci.jenkins.io/job/Infra/job/jenkins-infra/job/production/)
-
-This repository is the [r10k](https://github.com/adrienthebo/r10k) control
-repository for the [Jenkins](https://jenkins.io) project's own
-infrastructure.
+This repository hosts the Puppet code for the [Jenkins](https://jenkins.io) project's own infrastructure.
 
 ## Structure
 
@@ -39,28 +35,26 @@ This repo just manages and configures the deployments.
 
 ## Local development
 
-The amount of testing that can be done locally is as follows:
-
-* `bundle install` - To get the necessary gems to run tests locally, if you're
-  unfamiliar with Ruby development you may want to use [RVM](http://rvm.io/)
-  to create an isolated Ruby environment
-* `./check` - Will run the
-  [rspec-puppet](http://rspec-puppet) unit tests and the
-  [puppet-lint](http://puppet-lint.com) style validation. If you intend to run
-  the rspec-puppet over and over, use `rake spec_standalone` to avoid
-  re-initializing the Puppet module fixtures every time.
-
-### Pre-requisites for local development
+### Pre-requisites for Local Development
 
 * Ruby 2.6.x is required.
   * Please note that Ruby 2.7.x and 3.x have never been tested.
 * Bundler 1.17.x is required with the command line `bundle` installed and present in your `PATH`.
   * Please note that Bundler 2.x had never been tested
 * A bash-compliant shell is required.
   * `sh` has never been tested, neither Windows Cygwin Shell (but WSL is ok).
+* The command line `yq` in version 4.x is needed
 
 You can **always** check the Docker image that ci.jenkins.io uses to run the test harness for this project at <https://github.com/jenkins-infra/docker-inbound-agents/blob/main/ruby/Dockerfile> (Jenkins agent labelled with `ruby`).
 
+### Install Local Dependencies
+
+Run the script `./ci/00_setupgems.sh` to ensure that all the local dependencies are ready for local development, including:
+
+* Ruby Gems managed by `bundler` (through `Gemfile` and `Gemfile.lock`) to ensure development tools are available through `bundle exec <tool>` commands, installed to `vendor/gems`
+* Puppet modules retrieved from `./Puppetfile` and installed to `./modules`
+* Unit Tests fixtures generated from  `./Puppetfile` into `.fixtures.yml` but also other locations in `./spec/`
+
 ### Vagrant-based testing
 
 #### Running Acceptance Tests
@@ -94,13 +88,6 @@ To launch a test instance, `vagrant up ROLE` where `ROLE` is [one of the defined
 You can rerun puppet and execute tests with `vagrant provision ROLE` repeatedly while the VM is up and running.
 When it's all done, remove the instance the instance via `vagrant destroy ROLE`.
 
-### Updating dependencies
-
-For reasons that Tyler will hopefully clarify at some point, this module maintains
-the list of Puppet module dependencies in `Puppetfile` and `.fixtures.yml`. They
-need to be kept in sync. When you modify them, you can have the local environment
-reflect changes by running `bundle exec rake resolve`.
-
 ## Branching model
 
 The default branch of this repository is `production` which is where pull requests should be applied to by default.
@@ -146,6 +133,6 @@ See [this page](https://github.com/jenkins-infra/.github/blob/master/CONTRIBUTIN
 
 Channels:
 
-* `#jenkins-infra` on the [Freenode](http://freenode.net) IRC network
-* [INFRA project](https://issues.jenkins-ci.org/browse/INFRA) in JIRA.
-* [infra@lists.jenkins-ci.org](http://lists.jenkins-ci.org/mailman/listinfo/jenkins-infra)
+* `#jenkins-infra` on the [Libera Chat](https://libera.chat/guides) IRC network - see <https://www.jenkins.io/chat/>
+* [jenkins-infra/helpdesk Issue Tracker](https://github.com/jenkins-infra/helpdesk) in GitHub.
+* [jenkins-infra@groups.google.com](https://groups.google.com/g/jenkins-infra) mailing list

Rakefile

@@ -12,21 +12,21 @@ end
 
 PuppetLint::RakeTask.new :lint do |config|
   config.disable_checks = ['80chars',
-                           'class_parameter_defaults',
-                           'names_containing_dash']
+    'class_parameter_defaults',
+    'names_containing_dash']
   config.pattern = 'dist/**/*.pp'
   config.fail_on_warnings = true
 end
 
 desc 'Resolve all the dependencies'
 task :resolve do
-  # for reasons beyond me, we list dependencies in Puppetfile and .fixtures.yml
-  # we need to keep them in sync, and when we change them we need to run two commands
-  # to reflect those changes
+  # Cleanup
+  `rm -rf ./modules/ .fixtures.yml`
 
-  # this fills ./modules
-  `rm -rf ./modules/*`
-  `r10k puppetfile install`
+  # Generate the test fixtures from the Puppetfile and install them in ./spec/fixtures/ (modules, symlinks, etc.)
+  `generate-puppetfile -p ./Puppetfile --fixtures-only --ignore-comments`
+  # Remove jenkins_keys as it is a private repository
+  `yq --inplace 'del(.fixtures.repositories.jenkins_keys)' .fixtures.yml # Remove jenkins_keys as it is a private repository`
 
   # this fills ./spec/fixtures/modules
   Rake::Task['spec_clean'].invoke