Skip to content

Commit

Permalink
Make asset download feature usable by both "jobs post" and "isos post"
Browse files Browse the repository at this point in the history
  • Loading branch information
@okurz
okurz committed Sep 12, 2018
1 parent bc5d74b commit 2c27c90
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 69 deletions.
74 changes: 74 additions & 0 deletions lib/OpenQA/Utils.pm
View file
Expand Up @@ -19,6 +19,7 @@ require 5.002;
use Carp;
use IPC::Run();
use Mojo::URL;
use Mojo::Util 'url_unescape';
use Regexp::Common 'URI';
use Try::Tiny;
use Mojo::File 'path';
Expand Down Expand Up @@ -72,6 +73,8 @@ $VERSION = sprintf "%d.%03d", q$Revision: 1.12 $ =~ /(\d+)/g;
&asset_type_from_setting
&check_download_url
&check_download_whitelist
&create_downloads_list
&enqueue_download_jobs
&human_readable_size
&locate_asset
&job_groups_and_parents
Expand Down Expand Up @@ -715,6 +718,77 @@ sub check_download_whitelist {
return ();
}

sub create_downloads_list {
my ($args) = @_;
my %downloads = ();
for my $arg (keys %$args) {
next unless ($arg =~ /_URL$/);
# As this comes in from an API call, URL will be URI-encoded
# This obviously creates a vuln if untrusted users can POST
$args->{$arg} = url_unescape($args->{$arg});
my $url = $args->{$arg};
my $do_extract = 0;
my $short;
my $filename;
# if $args{FOO_URL} or $args{FOO_DECOMPRESS_URL} is set but $args{FOO}
# is not, we will set $args{FOO} (the filename of the downloaded asset)
# to the URL filename. This has to happen *before*
# generate_jobs so the jobs have FOO set
if ($arg =~ /_DECOMPRESS_URL$/) {
$do_extract = 1;
$short = substr($arg, 0, -15); # remove whole _DECOMPRESS_URL substring
}
else {
$short = substr($arg, 0, -4); # remove _URL substring
}
# We're only going to allow downloading of asset types. We also
# need this to determine the download location later
my $assettype = asset_type_from_setting($short);
unless ($assettype) {
OpenQA::Utils::log_debug("_URL downloading only allowed for asset types! $short is not an asset type");
next;
}
if (!$args->{$short}) {
$filename = Mojo::URL->new($url)->path->parts->[-1];
if ($do_extract) {
# if user wants to extract downloaded file, final filename
# will have last extension removed
$filename = fileparse($filename, qr/\.[^.]*/);
}
$args->{$short} = $filename;
if (!$args->{$short}) {
OpenQA::Utils::log_warning("Unable to get filename from $url. Ignoring $arg");
delete $args->{$short} unless $args->{$short};
next;
}
}
else {
$filename = $args->{$short};
}
# Find where we should download the file to
my $fullpath = locate_asset($assettype, $filename, mustexist => 0);

unless (-s $fullpath) {
# if the file doesn't exist, add the url/target path and extraction
# flag as a key/value pair to the %downloads hash
$downloads{$url} = [$fullpath, $do_extract];
}
}
return \%downloads;
}

sub enqueue_download_jobs {
my ($gru, $downloads, $job_ids) = @_;
return unless (%$downloads and @$job_ids);
# array of hashrefs job_id => id; this is what create needs
# to create entries in a related table (gru_dependencies)
my @jobsarray = map +{job_id => $_}, @$job_ids;
for my $url (keys %$downloads) {
my ($path, $do_extract) = @{$downloads->{$url}};
$gru->enqueue(download_asset => [$url, $path, $do_extract] => {priority => 20} => \@jobsarray);
}
}

sub send_job_to_worker {
my $ipc = OpenQA::IPC->ipc;
my $job = shift;
Expand Down
70 changes: 3 additions & 67 deletions lib/OpenQA/WebAPI/Controller/API/V1/Iso.pm
View file
Expand Up @@ -15,7 +15,6 @@

package OpenQA::WebAPI::Controller::API::V1::Iso;
use Mojo::Base 'Mojolicious::Controller';
use Mojo::Util 'url_unescape';
use File::Basename;

use OpenQA::Utils;
Expand Down Expand Up @@ -391,61 +390,8 @@ sub schedule_iso {

# Any arg name ending in _URL is special: it tells us to download
# the file at that URL before running the job
my %downloads = ();
for my $arg (keys %$args) {
next unless ($arg =~ /_URL$/);
# As this comes in from an API call, URL will be URI-encoded
# This obviously creates a vuln if untrusted users can POST
$args->{$arg} = url_unescape($args->{$arg});
my $url = $args->{$arg};
my $do_extract = 0;
my $short;
my $filename;
# if $args{FOO_URL} or $args{FOO_DECOMPRESS_URL} is set but $args{FOO}
# is not, we will set $args{FOO} (the filename of the downloaded asset)
# to the URL filename. This has to happen *before*
# generate_jobs so the jobs have FOO set
if ($arg =~ /_DECOMPRESS_URL$/) {
$do_extract = 1;
$short = substr($arg, 0, -15); # remove whole _DECOMPRESS_URL substring
}
else {
$short = substr($arg, 0, -4); # remove _URL substring
}
# We're only going to allow downloading of asset types. We also
# need this to determine the download location later
my $assettype = asset_type_from_setting($short);
unless ($assettype) {
OpenQA::Utils::log_debug("_URL downloading only allowed for asset types! $short is not an asset type");
next;
}
if (!$args->{$short}) {
$filename = Mojo::URL->new($url)->path->parts->[-1];
if ($do_extract) {
# if user wants to extract downloaded file, final filename
# will have last extension removed
$filename = fileparse($filename, qr/\.[^.]*/);
}
$args->{$short} = $filename;
if (!$args->{$short}) {
OpenQA::Utils::log_warning("Unable to get filename from $url. Ignoring $arg");
delete $args->{$short} unless $args->{$short};
next;
}
}
else {
$filename = $args->{$short};
}
# Find where we should download the file to
my $fullpath = locate_asset($assettype, $filename, mustexist => 0);

unless (-s $fullpath) {
# if the file doesn't exist, add the url/target path and extraction
# flag as a key/value pair to the %downloads hash
$downloads{$url} = [$fullpath, $do_extract];
}
}
my $jobs = $self->_generate_jobs($args);
my $downloads = create_downloads_list($args);
my $jobs = $self->_generate_jobs($args);

# XXX: take some attributes from the first job to guess what old jobs to
# cancel. We should have distri object that decides which attributes are
Expand Down Expand Up @@ -525,17 +471,7 @@ sub schedule_iso {
for my $job (@jobs) {
$job->calculate_blocked_by;
}

# enqueue gru jobs
if (%downloads and @successful_job_ids) {
# array of hashrefs job_id => id; this is what create needs
# to create entries in a related table (gru_dependencies)
my @jobsarray = map +{job_id => $_}, @successful_job_ids;
for my $url (keys %downloads) {
my ($path, $do_extract) = @{$downloads{$url}};
$self->gru->enqueue(download_asset => [$url, $path, $do_extract] => {priority => 20} => \@jobsarray);
}
}
enqueue_download_jobs($self->gru, $downloads, \@successful_job_ids);
};

try {
Expand Down
5 changes: 3 additions & 2 deletions lib/OpenQA/WebAPI/Controller/API/V1/Job.pm
View file
Expand Up @@ -19,7 +19,6 @@ use OpenQA::Utils;
use OpenQA::IPC;
use OpenQA::Jobs::Constants;
use OpenQA::Schema::Result::Jobs;
use OpenQA::Utils 'find_job';
use Try::Tiny;
use DBIx::Class::Timestamps 'now';
use Mojo::Asset::Memory;
Expand Down Expand Up @@ -189,7 +188,9 @@ sub create {
$self->emit_event('openqa_job_create', {id => $job->id, %params});
$json->{id} = $job->id;

# enqueue gru job
# enqueue gru jobs
my $downloads = create_downloads_list(\%params);
enqueue_download_jobs($self->gru, $downloads, [$job->id]);
$self->gru->enqueue(limit_assets => [] => {priority => 10});
$job->calculate_blocked_by;
wakeup_scheduler;
Expand Down

0 comments on commit 2c27c90

Please sign in to comment.