(RE-6628) Create new pack_signed_repo tasks

Our automation is going to need to handle more that one set of platforms to
promote to PE. Instead of adding seperate jobs for seperate pipelines (i.e.
one for master, one for stable etc.) and continuing to carry around static
lists on top of information we already have, this commit adds the ability to
use the platform information in build_defaults.yaml to identify what platforms
promote.

This is done through a new pack_all_signed_repos_individually task, to replace
the pack_signed_repo task (that created the need for the list in the jenkins
job).

to faciliate this, the following platform_repos data has been added to the
available data loaded in build_defaults.yaml, this data is in the following
form:

platform_repos:
  - name: platform-version-arch
    repo_location: repos/platform/repo/location
  - name: platfrom2-version-arch
    repo_location: repos/platform2/repo/location

lib/packaging.rb

@@ -15,6 +15,7 @@ module Pkg
   require 'packaging/nuget'
   require 'packaging/gem'
   require 'packaging/msi'
+  require 'packaging/repo'
 
   # Load configuration defaults
   Pkg::Config.load_defaults

lib/packaging/config/params.rb

@@ -125,6 +125,7 @@ module Pkg::Params
                   :pe_platforms,
                   :pe_version,
                   :pg_major_version,
+                  :platform_repos,
                   :pre_tar_task,
                   :pre_tasks,
                   :privatekey_pem,
@@ -324,5 +325,4 @@ module Pkg::Params
                   { :var => :gpg_name, :message => "
     DEPRECATED, 29-Jul-2014: 'gpg_name' has been replaced with 'gpg_key'.
                    Please update this field in your project_data.yaml" }]
-
 end

lib/packaging/repo.rb

@@ -0,0 +1,36 @@
+module Pkg::Repo
+
+  class << self
+    def create_signed_repo_archive(path_to_repo, name_of_archive, versioning)
+      tar = Pkg::Util::Tool.check_tool('tar')
+      Dir.chdir("pkg") do
+        if versioning == 'ref'
+          local_target = File.join(Pkg::Config.project, Pkg::Config.ref)
+        elsif versioning == 'version'
+          local_target = File.join(Pkg::Config.project, Pkg::Util::Version.get_dot_version)
+        end
+        Dir.chdir(local_target) do
+          if Pkg::Util::File.empty_dir?(path_to_repo)
+            if ENV['FAIL_ON_MISSING_TARGET'] == "true"
+              raise "ERROR: missing packages under #{path_to_repo}"
+            else
+              warn "Skipping #{name_of_archive} because #{path_to_repo} has no files"
+            end
+          else
+            puts "Archiving #{path_to_repo} as #{name_of_archive}"
+            stdout, _, _ = Pkg::Util::Execution.capture3("#{tar} --owner=0 --group=0 --create --gzip --file #{File.join('repos', "#{name_of_archive}.tar.gz")} #{path_to_repo}")
+            stdout
+          end
+        end
+      end
+    end
+
+    def create_all_repo_archives(project, versioning)
+      platforms = Pkg::Config.platform_repos
+      platforms.each do |platform|
+        archive_name = "#{project}-#{platform['name']}"
+        create_signed_repo_archive(platform['repo_location'], archive_name, versioning)
+      end
+    end
+  end
+end

spec/lib/packaging/repo_spec.rb

@@ -0,0 +1,97 @@
+# -*- ruby -*-
+require 'spec_helper'
+describe "#Pkg::Repo" do
+  let(:platform_repo_stub) do
+    [
+      {"name"=>"el-4-i386", "repo_location"=>"repos/el/4/**/i386"},
+      {"name"=>"el-5-i386", "repo_location"=>"repos/el/5/**/i386"},
+      {"name"=>"el-6-i386", "repo_location"=>"repos/el/6/**/i386"}
+    ]
+  end
+  describe "#create_signed_repo_archive" do
+    it "should change to the correct dir" do
+      allow(Pkg::Util::Tool).to receive(:check_tool).and_return("tarcommand")
+      allow(Pkg::Config).to receive(:project).and_return("project")
+      allow(Pkg::Util::Version).to receive(:get_dot_version).and_return("1.1.1")
+      allow(Pkg::Util::File).to receive(:empty_dir?).and_return(false)
+      allow(Pkg::Util::Execution).to receive(:capture3)
+
+      expect(Dir).to receive(:chdir).with("pkg").and_yield
+      expect(Dir).to receive(:chdir).with("project/1.1.1").and_yield
+      Pkg::Repo.create_signed_repo_archive("/path", "project-debian-6-i386", "version")
+    end
+
+    it "should use a ref if ref is specified as versioning" do
+      allow(Pkg::Util::Tool).to receive(:check_tool).and_return("tarcommand")
+      allow(Dir).to receive(:chdir).with("pkg").and_yield
+      allow(Pkg::Util::File).to receive(:empty_dir?).and_return(false)
+      allow(Pkg::Util::Execution).to receive(:capture3)
+
+      expect(Pkg::Config).to receive(:project).and_return("project")
+      expect(Pkg::Config).to receive(:ref).and_return("AAAAAAAAAAAAAAA")
+      expect(Dir).to receive(:chdir).with("project/AAAAAAAAAAAAAAA").and_yield
+      Pkg::Repo.create_signed_repo_archive("/path", "project-debian-6-i386", "ref")
+    end
+
+    it "should use dot versions if version is specified as versioning" do
+      allow(Pkg::Util::Tool).to receive(:check_tool).and_return("tarcommand")
+      allow(Dir).to receive(:chdir).with("pkg").and_yield
+      allow(Pkg::Util::File).to receive(:empty_dir?).and_return(false)
+      allow(Pkg::Util::Execution).to receive(:capture3)
+
+      expect(Pkg::Config).to receive(:project).and_return("project")
+      expect(Pkg::Util::Version).to receive(:get_dot_version).and_return("1.1.1")
+      expect(Dir).to receive(:chdir).with("project/1.1.1").and_yield
+      Pkg::Repo.create_signed_repo_archive("/path", "project-debian-6-i386", "version")
+    end
+
+    it "should fail if ENV['FAIL_ON_MISSING_TARGET'] is true and empty_dir? is also true" do
+      allow(Pkg::Util::Tool).to receive(:check_tool).and_return("tarcommand")
+      allow(Pkg::Config).to receive(:project).and_return("project")
+      allow(Pkg::Util::Version).to receive(:get_dot_version).and_return("1.1.1")
+      allow(Pkg::Util::Execution).to receive(:capture3)
+      allow(Dir).to receive(:chdir).with("pkg").and_yield
+      allow(Dir).to receive(:chdir).with("project/1.1.1").and_yield
+      allow(Pkg::Util::File).to receive(:empty_dir?).and_return(true)
+      ENV['FAIL_ON_MISSING_TARGET'] = "true"
+
+      expect{Pkg::Repo.create_signed_repo_archive("/path", "project-debian-6-i386", "version")}.to raise_error(RuntimeError, "ERROR: missing packages under /path")
+    end
+
+    it "should only warn if ENV['FAIL_ON_MISSING_TARGET'] is false and empty_dir? is true" do
+      allow(Pkg::Util::Tool).to receive(:check_tool).and_return("tarcommand")
+      allow(Pkg::Config).to receive(:project).and_return("project")
+      allow(Pkg::Util::Version).to receive(:get_dot_version).and_return("1.1.1")
+      allow(Pkg::Util::Execution).to receive(:capture3)
+      allow(Dir).to receive(:chdir).with("pkg").and_yield
+      allow(Dir).to receive(:chdir).with("project/1.1.1").and_yield
+      allow(Pkg::Util::File).to receive(:empty_dir?).and_return(true)
+      ENV['FAIL_ON_MISSING_TARGET'] = "false"
+
+      expect{Pkg::Repo.create_signed_repo_archive("/path", "project-debian-6-i386", "version")}.not_to raise_error
+    end
+
+    it "should invoke tar correctly" do
+      allow(Pkg::Util::Tool).to receive(:check_tool).and_return("tarcommand")
+      allow(Pkg::Config).to receive(:project).and_return("project")
+      allow(Pkg::Util::Version).to receive(:get_dot_version).and_return("1.1.1")
+      allow(Dir).to receive(:chdir).with("pkg").and_yield
+      allow(Dir).to receive(:chdir).with("project/1.1.1").and_yield
+      allow(Pkg::Util::File).to receive(:empty_dir?).and_return(false)
+
+      expect(Pkg::Util::Execution).to receive(:capture3).with("tarcommand --owner=0 --group=0 --create --gzip --file repos/project-debian-6-i386.tar.gz /path")
+      Pkg::Repo.create_signed_repo_archive("/path", "project-debian-6-i386", "version")
+    end
+  end
+
+  describe "#create_signed_repo_archive" do
+    it "should invoke create_signed_repo_archive correctly for multiple entries in platform_repos" do
+      allow(Pkg::Config).to receive(:platform_repos).and_return(platform_repo_stub)
+
+      expect(Pkg::Repo).to receive(:create_signed_repo_archive).with("repos/el/4/**/i386", "project-el-4-i386", "version")
+      expect(Pkg::Repo).to receive(:create_signed_repo_archive).with("repos/el/5/**/i386", "project-el-5-i386", "version")
+      expect(Pkg::Repo).to receive(:create_signed_repo_archive).with("repos/el/6/**/i386", "project-el-6-i386", "version")
+      Pkg::Repo.create_all_repo_archives("project", "version")
+    end
+  end
+end

tasks/nightly_repos.rake

@@ -54,24 +54,13 @@ namespace :pl do
       path_to_repo = args.path_to_repo or fail ":path_to_repo is a required argument for #{t}"
       name_of_archive = args.name_of_archive or fail ":name_of_archive is a required argument for #{t}"
       versioning = args.versioning or fail ":versioning is a required argument for #{t}"
-      tar = Pkg::Util::Tool.check_tool('tar')
-
-      Dir.chdir("pkg") do
-        if versioning == 'ref'
-          local_target = File.join(Pkg::Config.project, Pkg::Config.ref)
-        elsif versioning == 'version'
-          local_target = File.join(Pkg::Config.project, Pkg::Util::Version.get_dot_version)
-        end
+      Pkg::Repo.create_signed_repo_archive(path_to_repo, name_of_archive, versioning)
+    end
 
-        Dir.chdir(local_target) do
-          if Pkg::Util::File.empty_dir?(path_to_repo)
-            warn "Skipping #{name_of_archive} because it (#{path_to_repo}) has no files"
-          else
-            stdout, _, _ = Pkg::Util::Execution.capture3("#{tar} --owner=0 --group=0 --create --gzip --file #{File.join("repos", "#{name_of_archive}.tar.gz")} #{path_to_repo}")
-            stdout
-          end
-        end
-      end
+    task :pack_all_signed_repos_individually, [:name_of_archive, :versioning] => ["pl:fetch"] do |t, args|
+      name_of_archive = args.name_of_archive or fail ":name_of_archive is a required argument for #{t}"
+      versioning = args.versioning or fail ":versioning is a required argument for #{t}"
+      Pkg::Repo.create_all_repo_archives(name_of_archive, versioning)
     end
 
     # This is pretty similar to the 'pack_signed_repo' task. The difference here is that instead