diff --git a/etc/openqa/openqa.ini b/etc/openqa/openqa.ini index 21548494855..22a74b9ca91 100644 --- a/etc/openqa/openqa.ini +++ b/etc/openqa/openqa.ini @@ -11,6 +11,15 @@ ## space seaparated list of IPs or regular expressions that match IPs #allowed_hosts = 127.0.0.1 ::1 +## space separated list of domains from which asset download with +## _URL params is allowed. Matched at the end of the hostname in +## the URL. with these values downloads from opensuse.org, +## dl.fedoraproject.org, and a.b.c.opensuse.org are allowed; downloads +## from moo.net, dl.opensuse and fedoraproject.org.evil are not +## default is undefined, meaning asset download is *disabled*, you +## must set this option to enable it +#download_domains = fedoraproject.org opensuse.org + ## set if you have a local repo mirror #suse_mirror = http://FIXME diff --git a/lib/OpenQA/Scheduler/Scheduler.pm b/lib/OpenQA/Scheduler/Scheduler.pm index f3f7bf07597..95bb6eb813f 100644 --- a/lib/OpenQA/Scheduler/Scheduler.pm +++ b/lib/OpenQA/Scheduler/Scheduler.pm @@ -30,7 +30,7 @@ use Data::Dump qw/dd pp/; use Date::Format qw/time2str/; use DBIx::Class::Timestamps qw/now/; use DateTime; -use File::Spec::Functions qw/catfile/; +use File::Spec::Functions qw/catfile catdir/; use File::Temp qw/tempdir/; use Mojo::URL; use Mojo::Util 'url_unescape'; @@ -41,7 +41,7 @@ use OpenQA::Schema::Result::JobDependencies; use FindBin; use lib $FindBin::Bin; #use lib $FindBin::Bin.'Schema'; -use OpenQA::Utils qw/log_debug parse_assets_from_settings/; +use OpenQA::Utils qw/log_debug parse_assets_from_settings asset_type_from_setting/; use db_helpers qw/rndstr/; use OpenQA::IPC; @@ -1179,23 +1179,39 @@ sub job_schedule_iso { asset_register(%$a); } my $noobsolete = delete $args{_NOOBSOLETEBUILD}; - # ISOURL == ISO download. If the ISO already exists, skip the - # download step (below) entirely by leaving $isodlpath unset. - my $isodlpath; - if ($args{ISOURL}) { + # Any arg name ending in _URL is special: it tells us to download + # the file at that URL before running the job + my %downloads = (); + for my $arg (keys %args) { + next unless ($arg =~ /_URL$/); # As this comes in from an API call, URL will be URI-encoded - # This obviously creates a vuln if untrusted users can POST ISOs - $args{ISOURL} = url_unescape($args{ISOURL}); - # set $args{ISO} to the URL filename if we only got ISOURL. - # This has to happen *before* _generate_jobs so the jobs have - # ISO set - if (!$args{ISO}) { - $args{ISO} = Mojo::URL->new($args{ISOURL})->path->parts->[-1]; + # This obviously creates a vuln if untrusted users can POST + $args{$arg} = url_unescape($args{$arg}); + my $url = $args{$arg}; + # if $args{FOO_URL} is set but $args{FOO} is not, we will + # set $args{FOO} (the filename of the downloaded asset) to + # the URL filename. This has to happen *before* + # generate_jobs so the jobs have FOO set + my $short = substr($arg, 0, -4); + if (!$args{$short}) { + $args{$short} = Mojo::URL->new($url)->path->parts->[-1]; } - # full path to download target location - my $fulliso = catfile($OpenQA::Utils::isodir, $args{ISO}); - unless (-s $fulliso) { - $isodlpath = $fulliso; + # full path to download target location. We need to guess + # the asset type to know where to put it, using the same + # subroutine as parse_assets_from_settings + my $assettype = asset_type_from_setting($short); + # We're only going to allow downloading of asset types + unless ($assettype) { + OpenQA::Utils::log_warning("_URL downloading only allowed for asset types! $short is not an asset type"); + next; + } + my $dir = catdir($OpenQA::Utils::assetdir, $assettype); + my $fullpath = catfile($dir, $args{$short}); + + unless (-s $fullpath) { + # if the file doesn't exist, add the url/target path + # as a key/value pair to the %downloads hash + $downloads{$url} = $fullpath; } } my $jobs = _generate_jobs(%args); @@ -1263,18 +1279,21 @@ sub job_schedule_iso { }; # enqueue gru jobs - if ($isodlpath and @ids) { + if (%downloads and @ids) { # array of hashrefs job_id => id; this is what create needs # to create entries in a related table (gru_dependencies) my @jobsarray = map +{job_id => $_}, @ids; - schema->resultset('GruTasks')->create( - { - taskname => 'download_iso', - priority => 20, - args => [$args{ISOURL}, $isodlpath], - run_at => now(), - jobs => \@jobsarray, - }); + for my $url (keys %downloads) { + my $path = $downloads{$url}; + schema->resultset('GruTasks')->create( + { + taskname => 'download_asset', + priority => 20, + args => [$url, $path], + run_at => now(), + jobs => \@jobsarray, + }); + } } schema->resultset('GruTasks')->create( { diff --git a/lib/OpenQA/Schema/Result/Assets.pm b/lib/OpenQA/Schema/Result/Assets.pm index faa5d6759cb..01776c02c29 100644 --- a/lib/OpenQA/Schema/Result/Assets.pm +++ b/lib/OpenQA/Schema/Result/Assets.pm @@ -22,6 +22,8 @@ use OpenQA::Utils; use OpenQA::Scheduler::Scheduler 'job_notify_workers'; use Date::Format; use Mojo::UserAgent; +use File::Spec::Functions 'splitpath'; +use Try::Tiny; use db_helpers; @@ -114,14 +116,42 @@ sub ensure_size { # A GRU task...arguments are the URL to grab and the full path to save # it in. scheduled in job_schedule_iso() -sub download_iso { - my ($app, $args) = @_; - my ($url, $isodlpath) = @{$args}; +sub download_asset { + my ($app, $args) = @_; + my ($url, $dlpath) = @{$args}; # Bail if the dest file exists (in case multiple downloads of same ISO # are scheduled) - return if (-e $isodlpath); + return if (-e $dlpath); - OpenQA::Utils::log_debug("Downloading " . $url . " to " . $isodlpath . "..."); + my $dldir = (splitpath($dlpath))[1]; + unless (-w $dldir) { + OpenQA::Utils::log_error("download_asset: cannot write to $dldir"); + # we're not going to die because this is a gru task and we don't + # want to cause the Endless Gru Loop Of Despair, just return and + # let the jobs fail + job_notify_workers(); + return; + } + + # check URL is whitelisted for download. this should never fail; + # if it does, it means this task has been created without going + # through the ISO API controller, and that means either a code + # change we didn't think through or someone being evil + my @check = check_download_url($url, $app->config->{global}->{download_domains}); + if (@check) { + my ($status, $host) = @check; + if ($status == 2) { + OpenQA::Utils::log_error("download_asset: no hosts are whitelisted for asset download!"); + } + else { + OpenQA::Utils::log_error("download_asset: URL $url host $host is blacklisted!"); + } + OpenQA::Utils::log_error("**API MAY HAVE BEEN BYPASSED TO CREATE THIS TASK!**"); + job_notify_workers(); + return; + } + + OpenQA::Utils::log_debug("Downloading " . $url . " to " . $dlpath . "..."); my $ua = Mojo::UserAgent->new(max_redirects => 5); my $tx = $ua->build_tx(GET => $url); # Allow >16MiB downloads @@ -129,12 +159,20 @@ sub download_iso { $tx->res->max_message_size(0); $tx = $ua->start($tx); if ($tx->success) { - $tx->res->content->asset->move_to($isodlpath); + try { + $tx->res->content->asset->move_to($dlpath); + } + catch { + # again, we're trying not to die here, but log and return on fail + OpenQA::Utils::log_error("Error renaming temporary file to $dlpath: $_"); + job_notify_workers(); + return; + }; } else { # Clean up after ourselves. Probably won't exist, but just in case - OpenQA::Utils::log_debug("Downloading failed! Deleting files"); - unlink($isodlpath); + OpenQA::Utils::log_error("Downloading failed! Deleting files"); + unlink($dlpath); } # We want to notify workers either way: if we failed to download the ISO, # we want the jobs to run and fail. diff --git a/lib/OpenQA/Utils.pm b/lib/OpenQA/Utils.pm index 1864671c2b8..6af44cf9c65 100755 --- a/lib/OpenQA/Utils.pm +++ b/lib/OpenQA/Utils.pm @@ -4,6 +4,7 @@ require 5.002; use Carp; use IPC::Run(); +use Mojo::URL; require Exporter; our ($VERSION, @ISA, @EXPORT, @EXPORT_OK, %EXPORT_TAGS); @@ -27,6 +28,9 @@ $VERSION = sprintf "%d.%03d", q$Revision: 1.12 $ =~ /(\d+)/g; &parse_assets_from_settings &bugurl &bugref_to_href + &asset_type_from_setting + &check_download_url + &check_download_whitelist ); @@ -230,25 +234,32 @@ sub commit_git { return 1; } +sub asset_type_from_setting { + my ($setting) = @_; + if ($setting eq 'ISO' || $setting =~ /^ISO_\d$/) { + return 'iso'; + } + if ($setting =~ /^HDD_\d$/) { + return 'hdd'; + } + if ($setting =~ /^REPO_\d$/) { + return 'repo'; + } + if ($setting =~ /^ASSET_\d$/ || $setting eq 'KERNEL' || $setting eq 'INITRD') { + return 'other'; + } + # empty string if this doesn't look like an asset type + return ''; +} + sub parse_assets_from_settings { my ($settings) = (@_); my $assets = {}; for my $k (keys %$settings) { - if ($k eq 'ISO') { - $assets->{$k} = {type => 'iso', name => $settings->{$k}}; - } - if ($k =~ /^ISO_\d$/) { - $assets->{$k} = {type => 'iso', name => $settings->{$k}}; - } - if ($k =~ /^HDD_\d$/) { - $assets->{$k} = {type => 'hdd', name => $settings->{$k}}; - } - if ($k =~ /^REPO_\d$/) { - $assets->{$k} = {type => 'repo', name => $settings->{$k}}; - } - if ($k =~ /^ASSET_\d$/) { - $assets->{$k} = {type => 'other', name => $settings->{$k}}; + my $type = asset_type_from_setting($k); + if ($type) { + $assets->{$k} = {type => $type, name => $settings->{$k}}; } } @@ -274,5 +285,66 @@ sub bugref_to_href { return $text; } +sub check_download_url { + # Passed a URL and the download_domains whitelist from openqa.ini. + # Checks if the host of the URL is in the whitelist. Returns an + # array: (1, host) if there is a whitelist and the host is not in + # it, (2, host) if there is no whitelist, and () if we pass. This + # is used by check_download_whitelist below (and so indirectly by + # the Iso controller) and directly by the download_asset() Gru + # task subroutine. + my ($url, $whitelist) = @_; + my @okdomains; + if (defined $whitelist) { + @okdomains = split(/ /, $whitelist); + } + my $host = Mojo::URL->new($url)->host; + unless (@okdomains) { + return (2, $host); + } + my $ok = 0; + for my $okdomain (@okdomains) { + my $quoted = qr/$okdomain/; + $ok = 1 if ($host =~ /${quoted}$/); + } + if ($ok) { + return (); + } + else { + return (1, $host); + } +} + +sub check_download_whitelist { + # Passed the params hash ref for a job and the download_domains + # whitelist read from openqa.ini. Checks that all params ending + # in _URL (i.e. requesting asset download) specify URLs that are + # whitelisted. It's provided here so that we can run the check + # twice, once to return immediately and conveniently from the Iso + # controller, once again directly in the Gru asset download sub + # just in case someone somehow manages to bypass the API and + # create a gru task directly. On failure, returns an array of 4 + # items: the first is 1 if there was a whitelist at all or 2 if + # there was not, the second is the name of the param for which the + # check failed, the third is the URL, and the fourth is the host. + # On success, returns an empty array. + + my ($params, $whitelist) = @_; + my @okdomains; + if (defined $whitelist) { + @okdomains = split(/ /, $whitelist); + } + for my $param (keys %$params) { + next unless ($param =~ /_URL$/); + my $url = $$params{$param}; + my @check = check_download_url($url, $whitelist); + next unless (@check); + # if we get here, we got a failure + return ($check[0], $param, $url, $check[1]); + } + # empty list signals caller that check passed + return (); +} + 1; # vim: set sw=4 et: diff --git a/lib/OpenQA/WebAPI.pm b/lib/OpenQA/WebAPI.pm index d0227fb3e45..65c74061dd3 100644 --- a/lib/OpenQA/WebAPI.pm +++ b/lib/OpenQA/WebAPI.pm @@ -37,14 +37,15 @@ sub _read_config { my %defaults = ( global => { - appname => 'openQA', - base_url => undef, - branding => 'openSUSE', - allowed_hosts => undef, - suse_mirror => undef, - scm => undef, - hsts => 365, - audit_enabled => 1, + appname => 'openQA', + base_url => undef, + branding => 'openSUSE', + allowed_hosts => undef, + download_domains => undef, + suse_mirror => undef, + scm => undef, + hsts => 365, + audit_enabled => 1, }, auth => { method => 'OpenID', @@ -518,12 +519,12 @@ sub startup { ## JSON API ends here # - $self->gru->add_task(optipng => \&OpenQA::Schema::Result::Jobs::optipng); - $self->gru->add_task(reduce_result => \&OpenQA::Schema::Result::Jobs::reduce_result); - $self->gru->add_task(limit_assets => \&OpenQA::Schema::Result::Assets::limit_assets); - $self->gru->add_task(download_iso => \&OpenQA::Schema::Result::Assets::download_iso); - $self->gru->add_task(scan_old_jobs => \&OpenQA::Schema::Result::Needles::scan_old_jobs); - $self->gru->add_task(scan_needles => \&OpenQA::Schema::Result::Needles::scan_needles); + $self->gru->add_task(optipng => \&OpenQA::Schema::Result::Jobs::optipng); + $self->gru->add_task(reduce_result => \&OpenQA::Schema::Result::Jobs::reduce_result); + $self->gru->add_task(limit_assets => \&OpenQA::Schema::Result::Assets::limit_assets); + $self->gru->add_task(download_asset => \&OpenQA::Schema::Result::Assets::download_asset); + $self->gru->add_task(scan_old_jobs => \&OpenQA::Schema::Result::Needles::scan_old_jobs); + $self->gru->add_task(scan_needles => \&OpenQA::Schema::Result::Needles::scan_needles); # start workers checker $self->_workers_checker; diff --git a/lib/OpenQA/WebAPI/Controller/API/V1/Iso.pm b/lib/OpenQA/WebAPI/Controller/API/V1/Iso.pm index 8d9ff583563..4b7d3cea7b0 100644 --- a/lib/OpenQA/WebAPI/Controller/API/V1/Iso.pm +++ b/lib/OpenQA/WebAPI/Controller/API/V1/Iso.pm @@ -43,6 +43,24 @@ sub create { my %up_params = map { uc $_ => $params->{$_} } keys %$params; # restore URL encoded / my %params = map { $_ => $up_params{$_} =~ s@%2F@/@gr } keys %up_params; + + my @check = check_download_whitelist(\%params, $self->app->config->{global}->{download_domains}); + if (@check) { + my ($status, $param, $url, $host) = @check; + if ($status == 2) { + my $error = "Asset download requested but no domains whitelisted! Set download_domains"; + $self->app->log->debug("$param - $url"); + $self->res->message($error); + return $self->rendered(403); + } + else { + my $error = "Asset download requested from non-whitelisted host $host"; + $self->app->log->debug("$param - $url"); + $self->res->message($error); + return $self->rendered(403); + } + } + $self->emit_event('openqa_iso_create', \%params); my $ids = $ipc->scheduler('job_schedule_iso', \%params); diff --git a/lib/OpenQA/Worker/Common.pm b/lib/OpenQA/Worker/Common.pm index 876d7b05305..f75e4f54caf 100644 --- a/lib/OpenQA/Worker/Common.pm +++ b/lib/OpenQA/Worker/Common.pm @@ -23,7 +23,7 @@ use POSIX qw/uname/; use base qw/Exporter/; our @EXPORT = qw/$job $workerid $verbose $instance $worker_settings $pooldir $nocleanup $worker_caps $testresults $openqa_url - OPENQA_BASE OPENQA_SHARE ISO_DIR HDD_DIR ASSET_DIR STATUS_UPDATES_SLOW STATUS_UPDATES_FAST + OPENQA_BASE OPENQA_SHARE ISO_DIR HDD_DIR OTHER_DIR ASSET_DIR STATUS_UPDATES_SLOW STATUS_UPDATES_FAST add_timer remove_timer change_timer api_call verify_workerid register_worker ws_call/; @@ -50,8 +50,9 @@ use constant OPENQA_BASE => '/var/lib/openqa'; use constant OPENQA_SHARE => OPENQA_BASE . '/share'; use constant ASSET_DIR => OPENQA_SHARE . '/factory'; use constant { - ISO_DIR => ASSET_DIR . '/iso', - HDD_DIR => ASSET_DIR . '/hdd', + ISO_DIR => ASSET_DIR . '/iso', + HDD_DIR => ASSET_DIR . '/hdd', + OTHER_DIR => ASSET_DIR . '/other', }; use constant { STATUS_UPDATES_SLOW => 10, diff --git a/lib/OpenQA/Worker/Engines/isotovideo.pm b/lib/OpenQA/Worker/Engines/isotovideo.pm index 4838b3c1175..01130cb3656 100644 --- a/lib/OpenQA/Worker/Engines/isotovideo.pm +++ b/lib/OpenQA/Worker/Engines/isotovideo.pm @@ -90,6 +90,17 @@ sub engine_workit($) { } } + for my $otherkey (qw/KERNEL INITRD/) { + if (my $file = $job->{settings}->{$otherkey}) { + $file = join('/', OTHER_DIR, $file); + unless (-e $file) { + warn "$file does not exist!\n"; + return; + } + $job->{settings}->{$otherkey} = $file; + } + } + my $nd = $job->{settings}->{NUMDISKS} || 2; for my $i (1 .. $nd) { my $hdd = $job->{settings}->{"HDD_$i"} || undef; diff --git a/t/05-scheduler-iso.t b/t/05-scheduler-iso.t new file mode 100644 index 00000000000..3695f3560cc --- /dev/null +++ b/t/05-scheduler-iso.t @@ -0,0 +1,60 @@ +#!/usr/bin/env perl -w + +# Copyright (C) 2016 Red Hat +# Copyright (C) 2016 SUSE LLC +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Test job creation with job_create_iso. + +BEGIN { + unshift @INC, 'lib'; +} + +use strict; +use OpenQA::IPC; +use OpenQA::Scheduler; +use OpenQA::WebSockets; +use OpenQA::Test::Database; +use Net::DBus; +use Net::DBus::Test::MockObject; + +use Test::More tests => 4; + +# We need the fixtures so we have job templates +my $schema = OpenQA::Test::Database->new->create; + +# create Test DBus bus and service for fake WebSockets +my $ipc = OpenQA::IPC->ipc('', 1); +my $ws = OpenQA::WebSockets->new(); +my $sh = OpenQA::Scheduler->new(); + +# check we have no gru download tasks to start with +my @tasks = $schema->resultset("GruTasks")->search({taskname => 'download_asset'}); +ok(scalar @tasks == 0); + +# check a regular ISO post creates the expected number of jobs +my $ids = OpenQA::Scheduler::Scheduler::job_schedule_iso(DISTRI => 'opensuse', VERSION => '13.1', FLAVOR => 'DVD', ARCH => 'i586', ISO => 'openSUSE-13.1-DVD-i586-Build0091-Media.iso'); +ok($ids == 10); + +# Schedule download of an existing ISO; gru task should not be created +$ids = OpenQA::Scheduler::Scheduler::job_schedule_iso(DISTRI => 'opensuse', VERSION => '13.1', FLAVOR => 'DVD', ARCH => 'i586', ISO_URL => 'openSUSE-13.1-DVD-i586-Build0091-Media.iso'); +@tasks = $schema->resultset("GruTasks")->search({taskname => 'download_asset'}); +ok(scalar @tasks == 0); + +# Schedule download of a non-existing ISO; gru task should be created +$ids = OpenQA::Scheduler::Scheduler::job_schedule_iso(DISTRI => 'opensuse', VERSION => '13.1', FLAVOR => 'DVD', ARCH => 'i586', ISO_URL => 'nonexistent.iso'); +@tasks = $schema->resultset("GruTasks")->search({taskname => 'download_asset'}); +ok(scalar @tasks == 1);