SIMP-6514 Update Repo and unpackdvd changes (#22)

- fixed updaterepo to make sure permissions on the files in the
  repo are correct (not just repodata)
- updated unpack_dvd to separate noarch and arch rpms under SIMP
- added new options to unpack_dvd to allow user to set version dir
  because CentOS uses major version in .treeinfo
- allowed user to set what group to use to own files in unpack_dvd. This
   allows non-root testing of the script.
- add option in unpack_dvd to allow user to not link to major version
- updated scripts to use Puppet Ruby and then removed logic
  for old Ruby versions.  puppet-agent is already required by
  this package, so it makes sense to take advantage of the Ruby
  it provides.
- updated puppet-agent Requires to be greater than 5.0.0.
- Fleshed out README

SIMP-6514 #close
SIMP-6523 #close

README.md

@@ -3,3 +3,65 @@
 
 # simp-utils
 Utilities for managing SIMP
+
+## LDIFS
+
+Example ldif files to help manage LDAP. These ldifs are installed in
+`/usr/share/simp/ldifs`.
+
+## Scripts
+
+This section contains a brief description of the scripts installed under
+`/usr/local`.  See each script's help for more details.
+
+### /usr/local/bin/set_environment
+
+This is a YAML-based node classifier which can be used as a Puppet
+External Node Classifier (ENC).
+
+### /usr/local/bin/unpack_dvd
+
+Usage:  ``unpack_dvd [options] /path/to/dvd/to/unpack``
+
+This script unpacks either a SIMP ISO image or a distribution DVD to the specified
+directory or `/var/www/yum/` (default).
+
+The ``--help`` option gives a complete usage statement.
+
+### /usr/local/sbin/gen-ldap-update
+
+Usage:  ``gen-ldap-update``
+
+This is run on an LDAP server to generate an ldif file that can be used
+to update attributes in LDAP.
+
+### /usr/local/sbin/puppetlast
+
+Usage:  ``puppetlast [options]``
+
+``puppetlast`` queries PuppetDB and returns a list of nodes and the last
+time the catalog was compiled on each node.
+
+The ``--help`` option gives a complete usage statement.
+
+### /usr/local/sbin/simpenv
+
+Usage:
+  ``simpenv --list``
+  or
+  ``simpenv -n|-c|-l|-a [new|copy|link] [EXISTING_ENV] NEWENV``
+
+This script can be used to create or list environments in SIMP 6.4 or later.
+
+The ``--help`` option gives a complete usage statement.
+
+### /usr/local/sbin/updaterepos
+
+Usage:  ``updaterepos <repodir>``
+
+This will go into each directory under ``repodir`` that is not named
+``noarch`` and do the following:
+
+ - Update links to files under ``../noarch``
+ - Rebuild the ``repodata``
+ - Update the ``repodata`` to ensure it is readable by ``root:apache``

build/simp-utils.spec

@@ -6,8 +6,8 @@ License: Apache License, Version 2.0
 Group: Applications/System
 Source: %{name}-%{version}-%{release}.tar.gz
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
-Requires: puppet-agent >= 1.0.0
-Requires: mkisofs
+Requires: puppet-agent >= 5.0.0
+Requires: genisoimage
 Requires: rpm
 Requires: yum
 Requires: yum-utils
@@ -59,6 +59,23 @@ chmod -R u=rwx,g=rx,o=rx %{buildroot}/usr/local/*bin
 # Post uninstall stuff
 
 %changelog
+* Wed Jun 12 2019 Jeanne Greulich <jeanne.greulich@onyxpoint.com> - 6.2.2-0
+- Update updaterepo script to change permissions on the repo
+   files as well as repodata.
+- Update unpack_dvd script
+  - Make sure permissions on all directories containing RPMs for the
+    repo are correct.
+  - Only attempt to change ownership of files if run as root.
+  - Put `noarch` rpms under the `noarch` directory for the SIMP repo.
+  - Allow the user to specify the version directory for the OS, because
+    the CentOS `.treeinfo` file only contains the major OS version number.
+  - Added an option to allow user to not link the extracted files to the
+    major version.
+  - Added an option to change what group is used to own the files.
+  - Used puppet ruby instead of system ruby and removed Ruby 1.9 logic
+    and changed puppet-agent dependency to > 5.0
+- Updated the README
+
 * Mon Jun 03 2019 Liz Nemsick <lnemsick.simp@gmail.com> - 6.2.2-0
 - Update the path of SIMP's Puppet skeleton to
   /usr/share/simp/environment-skeleton/puppet. This is the correct

scripts/bin/set_environment

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/opt/puppetlabs/puppet/bin/ruby
 #
 # This is a YAML-based node classifier which can be used as a Puppet
 # External Node Classifier (ENC). It identifies the environment for

scripts/bin/unpack_dvd

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/opt/puppetlabs/puppet/bin/ruby
 
 #
 # This script unpacks either an ISO image or a DVD to the specified directory
@@ -7,8 +7,20 @@
 # It uses the 'isoinfo' utility to pull files off of the DVD so no root access
 # is requried for locally unpacking an ISO.
 #
+#  It uses the .treeinfo file to determine the OS family, version and arch
+#  and creates this OS directory (ie  RedHat/7.5/x86_64) and  a `SIMP` directory
+#  under the specified directory.
+#
+#  Any files under SIMP directory on the ISO are unpacked to the SIMP directory and
+#  all others under the OS directory
+#
+#  The OS repo will be created under an Updates directory.  Any RPM files found under
+#  any directory unpacked under the OS directory will be linked and included in the repo.
+#
+#  The ownership of the files is changed to root:apache if the script is run
+#  as root.
+#
 
-require 'rubygems'
 require 'optparse'
 require 'fileutils'
 require 'find'
@@ -54,15 +66,19 @@ class ProgressBar
   end
 end
 
-def update_yum_repo(repo)
+def update_yum_repo(repo, group)
+  repo_dirs = [ repo ]
+
   puts "Updating repo at #{repo}"
   FileUtils.mkdir_p(repo,{:mode => 0755})
   Dir.chdir(repo) do
     Find.find('../') do |path|
       Find.prune if File.basename(path) == File.basename(repo)
 
       if File.basename(path) =~ /.*\.rpm/ and not File.symlink?(File.basename(path)) then
+        full_path = File.dirname(File.expand_path(path))
         FileUtils.ln_sf(path,File.basename(path))
+        repo_dirs << full_path unless repo_dirs.include?(full_path)
       end
     end
 
@@ -93,29 +109,58 @@ def update_yum_repo(repo)
     end
   end
 
-  begin
-    FileUtils.chown_R('root','apache',repo)
-  rescue Exception => e
-    $stderr.puts("Warning: Could not change permissions on #{repo} to 'root:apache'.")
-    $stderr.puts(e)
+  if Process.uid == 0
+    repo_dirs.each  { |dir|
+      begin
+          FileUtils.chown_R('root',group, dir)
+      rescue Exception => e
+        $stderr.puts("Warning: Could not change permissions on #{dir} to root:#{group}.")
+        $stderr.puts(e)
+      end
+
+      begin
+        FileUtils.chmod_R('g+rX',dir)
+      rescue Exception => e
+        $stderr.puts("Warning: Could not change permissions on #{dir} to 'g+rX'.")
+        $stderr.puts(e)
+      end
+      begin
+        FileUtils.chmod('g+s',dir)
+      rescue Exception => e
+        $stderr.puts("Warning: Could  not set group id permission on #{dir} 'g+s'.")
+        $stderr.puts(e)
+      end
+    }
   end
+end
 
-  if Process.uid == 0
-    begin
-      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('1.9')
-        # Use Puppet to hack around the lack of symbolic modes in Ruby < 1.9
-        system(%{puppet resource file #{repo} recurse=true mode='g+srX' 2>&1 > /dev/null})
-      else
-        FileUtils.chmod_R('g+srX',repo)
+def sort_noarch(dir, arch)
+# dir = directory you are searching
+# arch = the architecture of the machine
+#
+#  This module sorts the noarch RPMs into the noarch
+#  directory.
+#
+  noarchdir = File.expand_path("../noarch", dir )
+  Dir.mkdir(noarchdir, 0750) unless Dir.exists?(noarchdir)
+  Dir.chdir(dir) do
+    Dir.glob('*.rpm').each do |rpmfile|
+      next if File.symlink?(rpmfile)
+      rpmarch = rpmfile.split('.')[-2]
+      unless [ arch, 'noarch'].include?(rpmarch)
+         rpmarch = `rpm -qp #{rpmfile} --queryformat '%{ARCH}' 2> /dev/null`
       end
-    rescue Exception => e
-      $stderr.puts("Warning: Could not change permissions on #{repo} to 'g+srX'.")
-      $stderr.puts(e)
+      FileUtils.mv(rpmfile, noarchdir, :force => true) if rpmarch.eql? 'noarch'
     end
   end
 end
 
 options = Hash.new
+# Set defaults
+options[:link] = true
+options[:group] = 'apache'
+
+# Get command line options
 opts = OptionParser.new do |opts|
   opts.banner = "Usage: #{$0} [options] /path/to/dvd/to/unpack"
 
@@ -131,6 +176,28 @@ opts = OptionParser.new do |opts|
     options[:dest] = dest.chomp
   end
 
+  opts.on("-v", "--version VERSION", "Override for the version in the .treeinfo file on the DVD.",
+    "  The CentOS DVDs often only have the major version and will overwrite",
+    "  what is in the existing major version directory. Use this option to specify",
+    "  a more specific version for the name of the directory if needed"
+    ) do |version|
+    options[:version] = version.chomp
+  end
+
+  opts.on("-n", "--nolink", "Do not Link the installed OS version to the major version",
+    "  By default it will link this newly installed DVD to the major version",
+    "  If you do not want this to happen use the -n option."
+    ) do |n|
+    options[:link] = false
+  end
+
+  opts.on("-g", "--group GROUP", "Change the group who will own the extracted files.",
+    "  Default group is apache",
+    "  Note: It does not check if the group exists at this time"
+    ) do |group|
+    options[:group] = group
+  end
+
   opts.on("-h", "--help", "Output a useful help message") do
     puts opts
     exit
@@ -147,6 +214,10 @@ end
 fail("Could not find a SIMP default output directory and no --dest option provided") unless options[:dest]
 fail("Destination directory does not exist") if not File.directory?(options[:dest])
 
+# Set nice names for options
+create_maj_version_link = options[:link]
+
+
 discattrs = {
   :family => nil,
   :version => nil,
@@ -194,6 +265,10 @@ discattrs.each_pair do |k,v|
   end
 end
 
+# Use the version in .treeinfo if no version has been specified"
+versiondir = options[:version] || discattrs[:version]
+
+
 puts "Starting to unpack #{discattrs[:path]}:"
 
 iso_list = %x{#{isoinfo} -Rl}.split("\n")
@@ -220,14 +295,13 @@ iso_toc = iso_toc - kill_dirs
 progress = ProgressBar.new(iso_toc.size)
 
 
-destdir = "#{options[:dest]}/#{discattrs[:family]}/#{discattrs[:version]}/#{discattrs[:arch]}/"
+destdir = "#{options[:dest]}/#{discattrs[:family]}/#{versiondir}/#{discattrs[:arch]}/"
 simpdir = "#{options[:dest]}/SIMP/#{discattrs[:arch]}/"
 
 iso_toc.each do |iso_entry|
   if iso_entry =~ /^\/SIMP/
     target = "#{simpdir}#{File.basename(iso_entry)}"
   else
-
     target = "#{destdir}#{iso_entry}"
   end
   begin
@@ -243,18 +317,21 @@ iso_toc.each do |iso_entry|
 end
 
 puts "Unpacking complete, updating yum repositories...."
-update_yum_repo("#{destdir}/Updates")
-update_yum_repo(simpdir) unless (simpdir == destdir)
+sort_noarch(simpdir,discattrs[:arch] )
+update_yum_repo("#{destdir}/Updates", options[:group])
+update_yum_repo(simpdir, options[:group]) unless (simpdir == destdir)
 puts "Repo creation complete"
 
-Dir.chdir("#{options[:dest]}/#{discattrs[:family]}") do
-  base_ver = discattrs[:version].split('.').first
-  if base_ver != discattrs[:version] then
-    from = discattrs[:version]
-    to = base_ver
-    puts "Linking #{from} to #{to}"
-    FileUtils.rm(to) if File.symlink?(to)
-    FileUtils.ln_sf(from,to)
+if create_maj_version_link then
+  Dir.chdir("#{options[:dest]}/#{discattrs[:family]}") do
+    maj_ver = discattrs[:version].split('.').first
+    if maj_ver != versiondir then
+      from = versiondir
+      to = maj_ver
+      puts "Linking #{from} to #{to}"
+      FileUtils.rm(to) if File.symlink?(to)
+      FileUtils.ln_sf(from,to)
+    end
   end
 end
 

scripts/sbin/gen-ldap-update

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/opt/puppetlabs/puppet/bin/ruby
 #
 # Print out a modify LDIF to set the attributes of the DNs of interest
 # to the values found in /etc/openldap/default.ldif.  This output should

scripts/sbin/updaterepos

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/opt/puppetlabs/puppet/bin/ruby
 
 # This class shamelessly borrowed from
 # http://www.software-testing.com.au/blog/2010/01/13/text-based-progress-bar-in-ruby-for-command-line-programs/
@@ -78,11 +78,8 @@ Dir.chdir(tgt_dir) {
         result = $?
         File.umask(0027)
 
-        FileUtils.chown_R('root','apache', 'repodata')
-# TODO The line below does not work on Ruby 1.8 (CentOS6).  Re-enable
-#      it when we drop support for CentOS6.
-#        FileUtils.chmod_R('g+srX','repodata')
-       %x[/bin/chmod -R g+srX #{File.join(tgt_dir, dir, 'repodata')}]
+        FileUtils.chown_R('root','apache', Dir.pwd)
+        FileUtils.chmod_R('g+srX', Dir.pwd)
 
         if result != 0 then
           $stderr.puts("Warning: There was an error running createrepo on #{tgt_dir}/#{dir}")

spec/acceptance/suites/default/nodesets

@@ -0,0 +1 @@
+../../nodesets

spec/acceptance/suites/default/updaterepos_spec.rb

@@ -169,6 +169,16 @@ def set_up_local_repo(host, repo_name)
       end
     end
 
+    it 'updaterepos should allow apache group access to directories and files' do
+      info = on(host, "ls -ld #{repo_dir(repo)}/x86_64").stdout.strip
+      expect(info).to match(/^drwxr\-[xs].*root\s+apache/)
+      info = on(host, "ls -l #{repo_dir(repo)}/x86_64/").stdout.strip
+      info.split("\n").each do |file_info|
+        next if file_info.match(/^total/)
+        expect(file_info).to match(/^.rw.r..*root\s+apache/)
+      end
+    end
+
     it 'updaterepos should be safely re-run' do
       on(host, command, :pty => true )
     end