Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

github_packages: support --keep-old #11093

Merged
merged 1 commit into from Apr 9, 2021
Merged

github_packages: support --keep-old #11093

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions Library/Homebrew/dev-cmd/pr-upload.rb
View file
Open in desktop
Expand Up @@ -164,6 +164,7 @@ def pr_upload
github_org = args.github_org || "homebrew"
github_packages = GitHubPackages.new(org: github_org)
github_packages.upload_bottles(bottles_hash,
keep_old: args.keep_old?,
dry_run: args.dry_run?,
warn_on_error: args.warn_on_upload_failure?)
else
Expand Down
96 changes: 72 additions & 24 deletions Library/Homebrew/github_packages.rb
View file
Open in desktop
Expand Up @@ -47,8 +47,15 @@ def initialize(org: "homebrew")
ENV["HOMEBREW_FORCE_HOMEBREW_ON_LINUX"] = "1" if @github_org == "homebrew" && !OS.mac?
end

sig { params(bottles_hash: T::Hash[String, T.untyped], dry_run: T::Boolean, warn_on_error: T::Boolean).void }
def upload_bottles(bottles_hash, dry_run:, warn_on_error:)
sig {
params(
bottles_hash: T::Hash[String, T.untyped],
keep_old: T::Boolean,
dry_run: T::Boolean,
warn_on_error: T::Boolean,
).void
}
def upload_bottles(bottles_hash, keep_old:, dry_run:, warn_on_error:)
user = Homebrew::EnvConfig.github_packages_user
token = Homebrew::EnvConfig.github_packages_token

Expand All @@ -75,7 +82,7 @@ def upload_bottles(bottles_hash, dry_run:, warn_on_error:)

bottles_hash.each do |formula_full_name, bottle_hash|
upload_bottle(user, token, skopeo, formula_full_name, bottle_hash,
dry_run: dry_run, warn_on_error: warn_on_error)
keep_old: keep_old, dry_run: dry_run, warn_on_error: warn_on_error)
end
end

Expand Down Expand Up @@ -194,7 +201,18 @@ def validate_schema!(schema_uri, json)
exit 1
end

def upload_bottle(user, token, skopeo, formula_full_name, bottle_hash, dry_run:, warn_on_error:)
def download(user, token, skopeo, image_uri, root, dry_run:)
puts
args = ["copy", "--all", image_uri.to_s, "oci:#{root}"]
if dry_run
puts "#{skopeo} #{args.join(" ")} --src-creds=#{user}:$HOMEBREW_GITHUB_PACKAGES_TOKEN"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not saying we should but: could moving this skopeo with or without dry run logic into a method (as it's used three times now)

Copy link
Member Author

@Bo98 Bo98 Apr 9, 2021 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The different args, even ones like print_stderr, make it a bit tricky.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fine with me 👍🏻

else
args << "--src-creds=#{user}:#{token}"
system_command!(skopeo, verbose: true, print_stdout: true, args: args)
end
end

def upload_bottle(user, token, skopeo, formula_full_name, bottle_hash, keep_old:, dry_run:, warn_on_error:)
formula_name = bottle_hash["formula"]["name"]

_, org, repo, = *bottle_hash["bottle"]["root_url"].match(URL_REGEX)
Expand All @@ -215,7 +233,10 @@ def upload_bottle(user, token, skopeo, formula_full_name, bottle_hash, dry_run:,
else
inspect_args << "--creds=#{user}:#{token}"
inspect_result = system_command(skopeo, print_stderr: false, args: inspect_args)
if inspect_result.status.success?
if keep_old
# Tag doesn't exist so ignore --keep-old
keep_old = false unless inspect_result.status.success?
elsif inspect_result.status.success?
if warn_on_error
opoo "#{image_uri} already exists, skipping upload!"
return
Expand All @@ -228,7 +249,11 @@ def upload_bottle(user, token, skopeo, formula_full_name, bottle_hash, dry_run:,
root = Pathname("#{formula_name}--#{version_rebuild}")
FileUtils.rm_rf root

write_image_layout(root)
if keep_old
download(user, token, skopeo, image_uri, root, dry_run: dry_run)
else
write_image_layout(root)
end

blobs = root/"blobs/sha256"
blobs.mkpath
Expand All @@ -253,24 +278,49 @@ def upload_bottle(user, token, skopeo, formula_full_name, bottle_hash, dry_run:,
end

created_date = bottle_hash["bottle"]["date"]
formula_annotations_hash = {
"com.github.package.type" => GITHUB_PACKAGE_TYPE,
"org.opencontainers.image.created" => created_date,
"org.opencontainers.image.description" => bottle_hash["formula"]["desc"],
"org.opencontainers.image.documentation" => documentation,
"org.opencontainers.image.license" => bottle_hash["formula"]["license"],
"org.opencontainers.image.ref.name" => version_rebuild,
"org.opencontainers.image.revision" => git_revision,
"org.opencontainers.image.source" => source,
"org.opencontainers.image.title" => formula_full_name,
"org.opencontainers.image.url" => bottle_hash["formula"]["homepage"],
"org.opencontainers.image.vendor" => org,
"org.opencontainers.image.version" => version,
}.reject { |_, v| v.blank? }

manifests = bottle_hash["bottle"]["tags"].map do |bottle_tag, tag_hash|
if keep_old
index = JSON.parse((root/"index.json").read)
image_index_sha256 = index["manifests"].first["digest"].delete_prefix("sha256:")
image_index = JSON.parse((blobs/image_index_sha256).read)
(blobs/image_index_sha256).unlink

formula_annotations_hash = image_index["annotations"]
manifests = image_index["manifests"]
else
formula_annotations_hash = {
"com.github.package.type" => GITHUB_PACKAGE_TYPE,
"org.opencontainers.image.created" => created_date,
"org.opencontainers.image.description" => bottle_hash["formula"]["desc"],
"org.opencontainers.image.documentation" => documentation,
"org.opencontainers.image.license" => bottle_hash["formula"]["license"],
"org.opencontainers.image.ref.name" => version_rebuild,
"org.opencontainers.image.revision" => git_revision,
"org.opencontainers.image.source" => source,
"org.opencontainers.image.title" => formula_full_name,
"org.opencontainers.image.url" => bottle_hash["formula"]["homepage"],
"org.opencontainers.image.vendor" => org,
"org.opencontainers.image.version" => version,
}.reject { |_, v| v.blank? }
manifests = []
end

processed_image_refs = Set.new
manifests.each do |manifest|
processed_image_refs << manifest["annotations"]["org.opencontainers.image.ref.name"]
end

manifests += bottle_hash["bottle"]["tags"].map do |bottle_tag, tag_hash|
bottle_tag = Utils::Bottles::Tag.from_symbol(bottle_tag.to_sym)

tag = GitHubPackages.version_rebuild(version, rebuild, bottle_tag.to_s)

if processed_image_refs.include?(tag)
puts
odie "A bottle JSON for #{bottle_tag} is present, but it is already in the image index!"
else
processed_image_refs << tag
end

local_file = tag_hash["local_filename"]
odebug "Uploading #{local_file}"

Expand Down Expand Up @@ -316,8 +366,6 @@ def upload_bottle(user, token, skopeo, formula_full_name, bottle_hash, dry_run:,
formulae_dir = tag_hash["formulae_brew_sh_path"]
documentation = "https://formulae.brew.sh/#{formulae_dir}/#{formula_name}" if formula_core_tap

tag = GitHubPackages.version_rebuild(version, rebuild, bottle_tag.to_s)

descriptor_annotations_hash = {
"org.opencontainers.image.ref.name" => tag,
"sh.brew.bottle.cpu.variant" => cpu_variant,
Expand Down