Skip to content

Commit

Permalink
Refs #33748 - simplify upload workflow and handle duplicate artifact
Browse files Browse the repository at this point in the history
  • Loading branch information
@jlsherrill
jlsherrill committed Nov 4, 2021
1 parent 3df3b3c commit 774c036
Show file tree
Hide file tree
Showing 8 changed files with 2,472 additions and 1,252 deletions.
67 changes: 25 additions & 42 deletions app/lib/actions/pulp3/orchestration/repository/import_upload.rb
View file
Expand Up @@ -4,58 +4,41 @@ module Pulp3
module Orchestration
module Repository
class ImportUpload < Pulp3::Abstract
# rubocop:disable Metrics/AbcSize
def plan(repository, smart_proxy, args)
file = {:filename => args.dig(:unit_key, :name), :sha256 => args.dig(:unit_key, :checksum) }
content_unit_href = args.dig(:unit_key, :content_unit_id)
docker_tag = (args.dig(:unit_type_id) == "docker_tag")
sequence do
if content_unit_href
duplicate_sha_path_content_list = ::Katello::Pulp3::PulpContentUnit.find_duplicate_unit(repository, args.dig(:unit_type_id), file, file[:sha256])
duplicate_content_href = duplicate_sha_path_content_list&.results&.first&.pulp_href

if duplicate_content_href
plan_self(:commit_output => [], :content_unit_href => duplicate_content_href)
action_output = plan_action(Pulp3::Repository::ImportUpload, duplicate_content_href, repository, smart_proxy).output
plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
else
duplicate_sha_artifact_list = ::Katello::Pulp3::Api::Core.new(smart_proxy).artifacts_api.list("sha256": file[:sha256])
duplicate_sha_artifact_href = duplicate_sha_artifact_list&.results&.first&.pulp_href
if duplicate_sha_artifact_href
artifact_output = plan_action(Pulp3::Repository::SaveArtifact,
file, repository, smart_proxy,
nil, args.dig(:unit_type_id),
artifact_href: duplicate_sha_artifact_href).output
content_unit_href = artifact_output[:pulp_tasks]
plan_self(:commit_output => nil, :artifact_output => artifact_output[:pulp_tasks])
action_output = plan_action(Pulp3::Repository::ImportUpload, content_unit_href, repository, smart_proxy).output
plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
end
end
elsif docker_tag
sequence do
if docker_tag
tag_manifest_output = plan_action(Pulp3::Repository::UploadTag,
repository,
smart_proxy,
args).output
repository,
smart_proxy,
args).output
plan_self(:commit_output => tag_manifest_output[:pulp_tasks])
plan_action(Pulp3::Repository::SaveVersion, repository, {force_fetch_version: true, tasks: tag_manifest_output[:pulp_tasks]})
else
commit_output = plan_action(Pulp3::Repository::CommitUpload,
repository,
smart_proxy,
"/pulp/api/v3/uploads/" + args.dig(:upload_id) + "/",
args.dig(:unit_key, :checksum)).output

artifact_output = plan_action(Pulp3::Repository::SaveArtifact,
file,
if content_unit_href
artifact_output = { :content_unit_href => content_unit_href }
commit_output = {}
else
commit_output = plan_action(Pulp3::Repository::CommitUpload,
repository,
smart_proxy,
commit_output[:pulp_tasks],
args.dig(:unit_type_id)).output
content_unit_href = artifact_output[:pulp_tasks]
plan_self(:commit_output => commit_output[:pulp_tasks], :artifact_output => artifact_output[:pulp_tasks])
action_output = plan_action(Pulp3::Repository::ImportUpload, content_unit_href, repository, smart_proxy).output
plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
"/pulp/api/v3/uploads/" + args.dig(:upload_id) + "/",
args.dig(:unit_key, :checksum)).output

artifact_output = plan_action(Pulp3::Repository::SaveArtifact,
file,
repository,
smart_proxy,
commit_output[:pulp_tasks],
args.dig(:unit_type_id)).output
end

plan_self(:commit_output => commit_output[:pulp_tasks], :artifact_output => artifact_output)
import_output = plan_action(Pulp3::Repository::ImportUpload, artifact_output, repository, smart_proxy).output
plan_action(Pulp3::Repository::SaveVersion, repository, tasks: import_output[:pulp_tasks])
end
end
end
Expand All @@ -65,7 +48,7 @@ def run
if input[:content_unit_href]
output[:content_unit_href] = input[:content_unit_href]
elsif input[:artifact_output]
output[:content_unit_href] = input[:artifact_output].last[:created_resources].first
output[:content_unit_href] = input[:artifact_output][:content_unit_href] || input[:artifact_output][:pulp_tasks].last[:created_resources].first
else
output[:content_unit_href] = nil
end
Expand Down
6 changes: 3 additions & 3 deletions app/lib/actions/pulp3/orchestration/repository/upload_content.rb
View file
Expand Up @@ -19,10 +19,10 @@ def plan(repository, smart_proxy, file, unit_type_id)
upload_action_output = plan_action(Pulp3::Repository::UploadFile, repository, smart_proxy, file[:path]).output
artifact_action_output = plan_action(Pulp3::Repository::SaveArtifact, file, repository, smart_proxy, upload_action_output[:pulp_tasks], unit_type_id).output
end
content_href = artifact_action_output[:pulp_tasks]

end
content_href ||= duplicate_content_href
action_output = plan_action(Pulp3::Repository::ImportUpload, content_href, repository, smart_proxy).output
artifact_action_output ||= {:content_unit_href => duplicate_content_href}
action_output = plan_action(Pulp3::Repository::ImportUpload, artifact_action_output, repository, smart_proxy).output
plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
plan_self(:subaction_output => action_output)
end
Expand Down
12 changes: 8 additions & 4 deletions app/lib/actions/pulp3/repository/import_upload.rb
View file
Expand Up @@ -4,15 +4,19 @@ module Actions
module Pulp3
module Repository
class ImportUpload < Pulp3::AbstractAsyncTask
def plan(content_unit, repository, smart_proxy)
plan_self(:content_unit => content_unit,
def plan(save_artifact_output, repository, smart_proxy)
plan_self(:save_artifact_output => save_artifact_output,
:repository_id => repository.id,
:smart_proxy_id => smart_proxy.id)
end

def invoke_external_task
content_unit = input[:content_unit]
content_unit_href = content_unit.is_a?(String) ? content_unit : content_unit.last[:created_resources].first
if input[:save_artifact_output][:pulp_tasks]&.any?
content_unit_href = input[:save_artifact_output][:pulp_tasks].last[:created_resources].first
else
content_unit_href = input[:save_artifact_output][:content_unit_href]
end

repo = ::Katello::Repository.find(input[:repository_id])
repo_backend_service = repo.backend_service(smart_proxy)
output[:content_unit_href] = content_unit_href
Expand Down
18 changes: 14 additions & 4 deletions app/lib/actions/pulp3/repository/save_artifact.rb
View file
Expand Up @@ -9,14 +9,24 @@ def plan(file, repository, smart_proxy, tasks, unit_type_id, options = {})
end

def invoke_external_task
repository = ::Katello::Repository.find(input[:repository_id])
artifact_href = input[:options][:artifact_href] || fetch_artifact_href
fail _("Content not uploaded to pulp") unless artifact_href
content_type = input[:unit_type_id]
content_backend_service = SmartProxy.pulp_primary.content_service(content_type)
output[:pulp_tasks] = [content_backend_service.content_api_create(relative_path: input[:options][:file_name],
artifact: artifact_href,
repository_id: input[:repository_id],
content_type: content_type)]

existing_content = ::Katello::Pulp3::PulpContentUnit.find_duplicate_unit(repository, input['unit_type_id'], {filename: input[:options][:file_name]}, input[:options][:sha256])
existing_content_href = existing_content&.results&.first&.pulp_href

if existing_content_href
output[:content_unit_href] = existing_content_href
[]
else
output[:pulp_tasks] = [content_backend_service.content_api_create(relative_path: input[:options][:file_name],
artifact: artifact_href,
repository_id: input[:repository_id],
content_type: content_type)]
end
end

def fetch_artifact_href
Expand Down
8 changes: 4 additions & 4 deletions test/actions/katello/repository_test.rb
View file
Expand Up @@ -290,7 +290,7 @@ class UploadFilesTest < TestBase
[{href: "demo_task/href"}],
"file")
assert_action_planed_with(action, ::Actions::Pulp3::Repository::ImportUpload,
[{href: "demo_task/artifact_href"}], repository_pulp3, proxy)
{pulp_tasks: [{href: "demo_task/artifact_href"}]}, repository_pulp3, proxy)
assert_action_planed_with(action, ::Actions::Pulp3::Repository::SaveVersion,
repository_pulp3,
tasks: [{href: "demo_task/version_href"}])
Expand All @@ -306,7 +306,7 @@ class UploadFilesTest < TestBase

plan_action action, repository_pulp3, proxy, {:path => file, :filename => 'puppet_module.tar.gz'}, 'file'
assert_action_planed_with(action, ::Actions::Pulp3::Repository::ImportUpload,
"demo_content/href", repository_pulp3, proxy)
{content_unit_href: "demo_content/href"}, repository_pulp3, proxy)
assert_action_planed_with(action, ::Actions::Pulp3::Repository::SaveVersion,
repository_pulp3,
tasks: [{href: "demo_task/version_href"}])
Expand Down Expand Up @@ -339,7 +339,7 @@ class UploadPythonTest < TestBase
[{href: "demo_task/href"}],
"python_package")
assert_action_planed_with(action, ::Actions::Pulp3::Repository::ImportUpload,
[{href: "demo_task/artifact_href"}], repository_python_pulp3, proxy)
{pulp_tasks: [{href: "demo_task/artifact_href"}]}, repository_python_pulp3, proxy)
assert_action_planed_with(action, ::Actions::Pulp3::Repository::SaveVersion,
repository_python_pulp3,
tasks: [{href: "demo_task/version_href"}])
Expand All @@ -355,7 +355,7 @@ class UploadPythonTest < TestBase

plan_action action, repository_python_pulp3, proxy, {:path => file, :filename => 'shelf_reader-0.1-py2-none-any.whl'}, 'python_package'
assert_action_planed_with(action, ::Actions::Pulp3::Repository::ImportUpload,
"demo_content/href", repository_python_pulp3, proxy)
{content_unit_href: "demo_content/href"}, repository_python_pulp3, proxy)
assert_action_planed_with(action, ::Actions::Pulp3::Repository::SaveVersion,
repository_python_pulp3,
tasks: [{href: "demo_task/version_href"}])
Expand Down

0 comments on commit 774c036

Please sign in to comment.