Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
574 changes: 4 additions & 570 deletions .rubocop_todo.yml

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion lib/packaging.rb
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
module Pkg

LIBDIR = File.expand_path(File.dirname(__FILE__))

$:.unshift(LIBDIR) unless
Expand Down
76 changes: 42 additions & 34 deletions lib/packaging/artifactory.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,11 @@
require 'packaging/artifactory/extensions'

module Pkg

# The Artifactory class
# This class provides automation to access the artifactory repos maintained
# by the Release Engineering team at Puppet. It has the ability to both push
# artifacts to the repos, and to retrieve them back from the repos.
class ManageArtifactory

# The Artifactory property that the artifactCleanup user plugin
# {https://github.com/jfrog/artifactory-user-plugins/tree/master/cleanup/artifactCleanup}
# uses to tell it to not clean a particular artifact
Expand Down Expand Up @@ -46,7 +44,6 @@ def initialize(project, project_version, opts = {})
attr_reader :project
attr_reader :project_version


# @param platform_tag [String] The platform tag string ('<platform>-<version>' or
# '<platform>-<version>-<arch>') for the repo we need
# information on. If generic information is needed, pass in `generic`
Expand All @@ -68,14 +65,12 @@ def location_for(platform_tag)
case format
when 'rpm'
toplevel_repo = 'rpm'
repo_subdirectories = File.join(
repo_subdirectories, "#{platform}-#{version}-#{architecture}")
repo_subdirectories = File.join(repo_subdirectories, "#{platform}-#{version}-#{architecture}")
when 'deb'
toplevel_repo = 'debian__local'
repo_subdirectories = File.join(repo_subdirectories, "#{platform}-#{version}")
when 'swix', 'dmg', 'svr4', 'ips'
repo_subdirectories = File.join(
repo_subdirectories, "#{platform}-#{version}-#{architecture}")
repo_subdirectories = File.join(repo_subdirectories, "#{platform}-#{version}-#{architecture}")
when 'msi'
repo_subdirectories = File.join(repo_subdirectories, "#{platform}-#{architecture}")
end
Expand Down Expand Up @@ -106,7 +101,7 @@ def platform_specific_data(platform_tag)
package_format: package_format,
repo_name: repo_name,
repo_subdirectories: repo_subdirectories,
full_artifactory_path: full_artifactory_path
full_artifactory_path: full_artifactory_path,
}
end

Expand All @@ -118,7 +113,7 @@ def deb_list_contents(platform_tag)
data = platform_specific_data(platform_tag)
if data[:package_format] == 'deb'
return format("deb %s/%s %s %s", @artifactory_uri, data[:repo_name],
data[:codename], data[:repo_subdirectories])
data[:codename], data[:repo_subdirectories])

end
raise "Error: the platform '#{platform_tag}' is not an apt-based system."
Expand All @@ -142,6 +137,7 @@ def rpm_repo_contents(platform_tag)
#repo_gpgcheck=1
DOC
end

raise "The platform '#{platform_tag}' is not a yum-based system"
end

Expand Down Expand Up @@ -190,19 +186,15 @@ def deploy_properties(platform_tag, file_name)
# current iteration, the hash isn't formatted properly and the attempt to
# deploy to Artifactory bails out. I'm leaving this in so that we at least
# have multiple places to remind us that it needs to happen.
#properties_hash = Pkg::Config.config_to_hash
# properties_hash = Pkg::Config.config_to_hash
properties_hash = {}
if data[:package_format] == 'deb'
architecture = data[:architecture]
# set arch correctly for noarch packages
if file_name =~ /_all\.deb$/
architecture = 'all'
end
properties_hash.merge!({
'deb.distribution' => data[:codename],
'deb.component' => data[:repo_subdirectories],
'deb.architecture' => architecture,
})
properties_hash.merge!({ 'deb.distribution' => data[:codename], 'deb.component' => data[:repo_subdirectories], 'deb.architecture' => architecture })
end
properties_hash
end
Expand All @@ -213,10 +205,7 @@ def deploy_properties(platform_tag, file_name)
# @return [Array] Artifactory search for packages named with the basename of the package.
def package_exists_on_artifactory?(package)
check_authorization
found_artifacts = Artifactory::Resource::Artifact.search(
name: File.basename(package),
artifactory_uri: @artifactory_uri
)
found_artifacts = Artifactory::Resource::Artifact.search(name: File.basename(package), artifactory_uri: @artifactory_uri)
return false if found_artifacts.empty?

return found_artifacts
Expand Down Expand Up @@ -270,7 +259,7 @@ def deploy_package(package)
data[:repo_name],
remote_path,
deploy_properties(platform_tag, File.basename(package)),
headers
headers,
)
end

Expand Down Expand Up @@ -346,11 +335,13 @@ def promote_package(pkg, ref, platform_tag, repository, debian_component = nil)
artifact_names = all_package_names(yaml_data[:platform_data], platform_tag)
artifact_names.each do |artifact_name|
artifact_search_results = Artifactory::Resource::Artifact.search(
name: artifact_name, :artifactory_uri => @artifactory_uri)
name: artifact_name, :artifactory_uri => @artifactory_uri,
)

if artifact_search_results.empty?
raise "Error: could not find PKG=#{pkg} at REF=#{ref} for #{platform_tag}"
end

artifact_to_promote = artifact_search_results[0]

# This makes an assumption that we're using some consistent repo names
Expand All @@ -373,7 +364,8 @@ def promote_package(pkg, ref, platform_tag, repository, debian_component = nil)
unless properties.nil?
artifacts = Artifactory::Resource::Artifact.search(
name: artifact_name,
artifactory: @artifactory_uri)
artifactory: @artifactory_uri,
)
promoted_artifact = artifacts.select do |artifact|
artifact.download_uri =~ %r{#{promotion_path}}
end.first
Expand Down Expand Up @@ -410,7 +402,8 @@ def download_packages(staging_directory, manifest, remote_path = '')
artifacts = Artifactory::Resource::Artifact.checksum_search(
md5: "#{info["md5"]}",
repos: ["rpm_enterprise__local", "debian_enterprise__local"],
name: filename)
name: filename,
)
artifact_to_download = artifacts.select do |artifact|
artifact.download_uri.include? remote_path
end.first
Expand All @@ -428,7 +421,8 @@ def download_packages(staging_directory, manifest, remote_path = '')
artifacts = Artifactory::Resource::Artifact.checksum_search(
md5: "#{info["md5"]}",
repos: ["rpm_enterprise__local", "debian_enterprise__local"],
name: filename)
name: filename,
)
artifact_to_download = artifacts.select do |artifact|
artifact.download_uri.include? remote_path
end.first
Expand All @@ -450,7 +444,6 @@ def download_packages(staging_directory, manifest, remote_path = '')
end
end


# Ship PE tarballs to specified artifactory repo and paths
# @param local_tarball_directory [String] the local directory containing the tarballs
# @param target_repo [String] the artifactory repo to ship the tarballs to
Expand All @@ -461,10 +454,12 @@ def ship_pe_tarballs(local_tarball_directory, target_repo, ship_paths)
ship_paths.each do |path|
Dir.foreach(local_tarball_directory) do |pe_tarball|
next if pe_tarball == '.' || pe_tarball == ".."

begin
puts "Uploading #{pe_tarball} to #{target_repo}/#{path}#{pe_tarball}"
artifact = Artifactory::Resource::Artifact.new(
local_path: "#{local_tarball_directory}/#{pe_tarball}")
local_path: "#{local_tarball_directory}/#{pe_tarball}",
)
artifact.upload(target_repo, "#{path}#{pe_tarball}")
rescue Errno::EPIPE
STDERR.puts "Warning: Could not upload #{pe_tarball} to #{target_repo}/#{path}. Skipping."
Expand All @@ -485,6 +480,7 @@ def ship_pe_tarballs(local_tarball_directory, target_repo, ship_paths)
# "X-Checksum-Md5" and "X-Checksum-Sha1" are typical
def upload_file(local_path, target_repo, target_path, properties = {}, headers = {})
fail "Error: Couldn't find file at #{local_path}." unless File.exist? local_path

check_authorization
artifact = Artifactory::Resource::Artifact.new(local_path: local_path)
full_upload_path = File.join(target_path, File.basename(local_path))
Expand Down Expand Up @@ -516,11 +512,11 @@ def prevent_artifact_cleanup(repo, directory, pe_build_version)

all_artifacts = Artifactory::Resource::Artifact.pattern_search(
repo: repo,
pattern: all_artifacts_pattern
pattern: all_artifacts_pattern,
)
latest_artifacts = Artifactory::Resource::Artifact.pattern_search(
repo: repo,
pattern: latest_artifacts_pattern
pattern: latest_artifacts_pattern,
)

# Clear cleanup.skip on all artifacts in directory
Expand Down Expand Up @@ -560,6 +556,7 @@ def download_artifact(artifact_name, repo, path, target: '.', filename: nil)
filename ||= artifact_name
artifacts = search_with_path(artifact_name, repo, path)
return nil if artifacts.empty?

# Only download the first of the artifacts since we're saving them to
# the same location anyways
artifacts.first.download(target, filename: filename)
Expand All @@ -573,10 +570,12 @@ def download_artifact(artifact_name, repo, path, target: '.', filename: nil)
def download_final_pe_tarballs(pe_version, repo, remote_path, local_path)
check_authorization
artifacts = Artifactory::Resource::Artifact.search(
name: pe_version, repos: repo, exact_match: false)
name: pe_version, repos: repo, exact_match: false,
)
artifacts.each do |artifact|
next unless artifact.download_uri.include? remote_path
next if artifact.download_uri.include? "-rc"

artifact.download(local_path)
end
end
Expand All @@ -603,10 +602,12 @@ def download_beta_pe_tarballs(beta_tag, repo, remote_path, local_path)
def copy_final_pe_tarballs(pe_version, repo, remote_path, target_path)
check_authorization
final_tarballs = Artifactory::Resource::Artifact.search(
name: pe_version, repos: repo, exact_match: false)
name: pe_version, repos: repo, exact_match: false,
)
final_tarballs.each do |artifact|
next unless artifact.download_uri.include? remote_path
next if artifact.download_uri.include? "-rc"

filename = File.basename(artifact.download_uri)
# Artifactory does NOT like when you use `File.join`, so let's concatenate!
full_target_path = "#{repo}/#{target_path}/#{filename}"
Expand All @@ -629,7 +630,7 @@ def copy_artifact(artifact, target_repo, target_path, target_debian_component =
begin
artifact.copy(artifactory_target_path)
rescue Artifactory::Error::HTTPError
STDERR.puts "Warning: Could not copy #{artifactory_target_path}. "\
STDERR.puts "Warning: Could not copy #{artifactory_target_path}. " \
"Source and destination are the same. Skipping."
end

Expand Down Expand Up @@ -663,10 +664,12 @@ def populate_pe_repos(manifest, target_path)
artifact = Artifactory::Resource::Artifact.checksum_search(
md5: "#{info["md5"]}",
repos: ["rpm_enterprise__local", "debian_enterprise__local"],
name: filename).first
name: filename,
).first
if artifact.nil?
raise "Error: could not find package #{filename} with md5sum #{info["md5"]}"
end

copy_artifact(artifact, artifact.repo, "#{target_path}/#{dist}/#{filename}")
end
end
Expand Down Expand Up @@ -700,11 +703,14 @@ def remove_promoted_packages(manifest, remote_path, package, repos)
manifest.each do |dist, packages|
packages.each do |package_name, info|
next unless package_name == package

filename = info["filename"]
artifacts = Artifactory::Resource::Artifact.checksum_search(
md5: "#{info["md5"]}", repos: repos, name: filename)
md5: "#{info["md5"]}", repos: repos, name: filename,
)
artifacts.each do |artifact|
next unless artifact.download_uri.include? remote_path

puts "Info: removing reverted package #{artifact.download_uri}"
artifact.delete
end
Expand All @@ -723,9 +729,11 @@ def purge_copied_pe_tarballs(tarball_path, pe_repo)
check_authorization
Dir.foreach("#{tarball_path}/") do |pe_tarball|
next if pe_tarball == '.' || pe_tarball == ".."

md5 = Digest::MD5.file("#{tarball_path}/#{pe_tarball}").hexdigest
artifacts_to_delete = Artifactory::Resource::Artifact.checksum_search(
md5: md5, repos: pe_repo, name: pe_tarball)
md5: md5, repos: pe_repo, name: pe_tarball,
)
next if artifacts_to_delete.nil?

begin
Expand Down
29 changes: 17 additions & 12 deletions lib/packaging/config.rb
Original file line number Diff line number Diff line change
Expand Up @@ -79,13 +79,13 @@ def platform_data
Pkg::Util::Net.check_host_ssh([self.builds_server]).empty?

dir = "/opt/jenkins-builds/#{self.project}/#{self.ref}"
cmd = "if [ -s \"#{dir}/artifacts\" ]; then cd #{dir};"\
cmd = "if [ -s \"#{dir}/artifacts\" ]; then cd #{dir};" \
"find ./artifacts -mindepth 2 -type f; fi"
artifacts, _ = Pkg::Util::Net.remote_execute(
self.builds_server,
cmd,
{ capture_output: true }
)
artifacts, = Pkg::Util::Net.remote_execute(
self.builds_server,
cmd,
{ capture_output: true },
)

artifacts = artifacts.split("\n")
data = {}
Expand Down Expand Up @@ -131,12 +131,12 @@ def platform_data

case package_format
when 'deb'
repo_config = "../repo_configs/deb/pl-#{self.project}-#{self.ref}-"\
repo_config = "../repo_configs/deb/pl-#{self.project}-#{self.ref}-" \
"#{Pkg::Platforms.get_attribute(tag, :codename)}.list"
when 'rpm'
# Using original_tag here to not break legacy fedora repo targets
unless tag.include? 'aix'
repo_config = "../repo_configs/rpm/pl-#{self.project}-"\
repo_config = "../repo_configs/rpm/pl-#{self.project}-" \
"#{self.ref}-#{original_tag}.repo"
end
when 'swix', 'svr4', 'ips', 'dmg', 'msi'
Expand Down Expand Up @@ -227,8 +227,8 @@ def print_config
# the debian changelog.
#
def cow_list
self.cows.split(' ').map do
|cow| cow.split('-')[1]
self.cows.split(' ').map do |cow|
cow.split('-')[1]
end.uniq.join(' ')
end

Expand All @@ -253,11 +253,11 @@ def load_default_configs
got_config = false
default_project_data = {
path: File.join(@project_root, "ext", "project_data.yaml"),
required: false
required: false,
}
default_build_defaults = {
path: File.join(@project_root, "ext", "build_defaults.yaml"),
required: true
required: true,
}

[default_project_data, default_build_defaults].each do |config|
Expand Down Expand Up @@ -433,6 +433,7 @@ def perform_validations
def string_to_array(str)
delimiters = /[,\s;]/
return str if str.respond_to?('each')

str.split(delimiters).reject { |s| s.empty? }.map { |s| s.strip }
end

Expand All @@ -452,9 +453,11 @@ def mock_to_dist_version_arch(mock)
def deb_build_targets
if self.vanagon_project
fail "ERROR: Could not find any deb targets. Try adding `deb_targets` to your build_defaults.yaml. If you don't want to build any debs, set this to an empty string." unless self.deb_targets

self.deb_targets.split(' ')
else
fail "ERROR: Could not find any deb targets. Try adding `cows` to your build_defaults.yaml. If you don't want to build any debs, set this to an empty string." unless self.cows

self.cows.split(' ').map do |cow|
codename, arch = self.cow_to_codename_arch(cow)
"#{codename}-#{arch}"
Expand All @@ -465,9 +468,11 @@ def deb_build_targets
def rpm_build_targets
if self.vanagon_project
fail "ERROR: Could not find any rpm targets. Try adding `rpm_targets` to your build_defaults.yaml. If you don't want to build any rpms, set this to an empty string." unless self.rpm_targets

self.rpm_targets.split(' ')
else
fail "ERROR: Could not find any rpm targets. Try adding `final_mocks` to your build_defaults.yaml. If you don't want to build any rpms, set this to an empty string." unless self.final_mocks

self.final_mocks.split(' ').map do |mock|
platform, version, arch = self.mock_to_dist_version_arch(mock)
"#{platform}-#{version}-#{arch}"
Expand Down
Loading