Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Parallel gru jobs #1984

Merged
merged 3 commits into from
Feb 1, 2019
Merged

Parallel gru jobs #1984

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
feef172
Lift the parallel job limit for the gru worker and add more fine grai…
kraih Jan 31, 2019
ffcddb2
Remove image migration jobs that were only used for very old migrations
kraih Jan 31, 2019
334f53a
Remove SQL files for 2017
kraih Feb 1, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 0 additions & 51 deletions dbicdh/_common/upgrade/45-46/01-migrate-images.pl
View file
Open in desktop

This file was deleted.

4 changes: 4 additions & 0 deletions lib/OpenQA/Task/Job/Limit.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ sub register {
sub _limit {
my ($app, $job) = @_;

# prevent multiple limit_results_and_logs tasks to run in parallel
return $job->finish('Previous limit_results_and_logs job is still active')
unless my $guard = $app->minion->guard('limit_results_and_logs_task', 7200);

# create temporary job group outside of DB to collect
# jobs without job_group_id
$app->db->resultset('JobGroups')->new({})->limit_results_and_logs;
Expand Down
127 changes: 0 additions & 127 deletions lib/OpenQA/Task/Job/Modules.pm
View file
Open in desktop

This file was deleted.

6 changes: 5 additions & 1 deletion lib/OpenQA/Task/Needle/Scan.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,11 @@ sub register {
}

sub _needles {
my ($app, $minion, $args) = @_;
my ($app, $job, $args) = @_;

# prevent multiple scan_needles tasks to run in parallel
return $job->finish('Previous scan_needles job is still active')
unless my $guard = $app->minion->guard('limit_scan_needles_task', 7200);

my $dirs = $app->db->resultset('NeedleDirs');

Expand Down
11 changes: 9 additions & 2 deletions lib/OpenQA/Task/Screenshot/Scan.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,11 @@ sub _list_images_subdir {

# gru task to scan XXX subdirectory
sub _scan_images {
my ($app, $minion, $args) = @_;
my ($app, $job, $args) = @_;

# prevent multiple scan_images* tasks to run in parallel
return $job->retry({delay => 30})
unless my $guard = $app->minion->guard('limit_scan_images_task', 3600);

return unless $args->{prefix};
my $dh;
Expand Down Expand Up @@ -92,8 +96,11 @@ sub _scan_images {

# gru task - scan testresults and add them to Screenshotlinks
sub _scan_images_links {
my ($app, $minion, $args) = @_;
my ($app, $job, $args) = @_;

# prevent multiple scan_images* tasks to run in parallel
return $job->retry({delay => 30})
unless my $guard = $app->minion->guard('limit_scan_images_task', 3600);

my $schema = OpenQA::Scheduler::Scheduler::schema();
my $jobs = $schema->resultset("Jobs")->search(
Expand Down
4 changes: 0 additions & 4 deletions lib/OpenQA/WebAPI/Command/gru/run.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,6 @@ sub run {
$minion->on(
worker => sub {
my ($minion, $worker) = @_;

# Only one job can run at a time for now (until all Gru tasks are parallelism safe)
$worker->status->{jobs} = 1;

$worker->on(
dequeue => sub {
my ($worker, $job) = @_;
Expand Down
8 changes: 2 additions & 6 deletions lib/OpenQA/WebAPI/Plugin/Gru.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,8 @@ sub register_tasks {

my $app = $self->app;
$app->plugin($_)
for (
qw(OpenQA::Task::Asset::Download OpenQA::Task::Asset::Limit),
qw(OpenQA::Task::Job::Limit OpenQA::Task::Job::Modules),
qw(OpenQA::Task::Needle::Scan),
qw(OpenQA::Task::Screenshot::Scan)
);
for (qw(OpenQA::Task::Asset::Download OpenQA::Task::Asset::Limit OpenQA::Task::Job::Limit),
qw(OpenQA::Task::Needle::Scan OpenQA::Task::Screenshot::Scan));
}

sub register {
Expand Down
60 changes: 0 additions & 60 deletions t/14-grutasks.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,66 +309,6 @@ subtest 'limit_results_and_logs gru task cleans up logs' => sub {
ok(!-e $filename, 'file got cleaned');
};

subtest 'migrate_images' => sub {
File::Path::remove_tree('t/images/aa7/');
File::Path::make_path('t/data/openqa/images/aa/.thumbs');
copy(
't/images/347/da6/.thumbs/61d0c3faf37d49d33b6fc308f2.png',
't/data/openqa/images/aa/.thumbs/7da661d0c3faf37d49d33b6fc308f2.png'
);
copy(
't/images/347/da6/61d0c3faf37d49d33b6fc308f2.png',
't/data/openqa/images/aa//7da661d0c3faf37d49d33b6fc308f2.png'
);
ok(!-l 't/data/openqa/images/aa/.thumbs/7da661d0c3faf37d49d33b6fc308f2.png', 'no link yet');

run_gru('migrate_images' => {prefix => 'aa'});
ok(-l 't/data/openqa/images/aa/.thumbs/7da661d0c3faf37d49d33b6fc308f2.png', 'now link');
ok(-e 't/data/openqa/images/aa7/da6/.thumbs/61d0c3faf37d49d33b6fc308f2.png', 'file moved');

File::Path::remove_tree('t/images/aa7/');
};

subtest 'relink_testresults' => sub {
File::Path::make_path('t/data/openqa/images/34/.thumbs');
symlink(
'../../../images/347/da6/.thumbs/61d0c3faf37d49d33b6fc308f2.png',
't/data/openqa/images/34/.thumbs/7da661d0c3faf37d49d33b6fc308f2.png'
);

# setup
unlink('t/data/openqa/testresults/00099/00099937-opensuse-13.1-DVD-i586-Build0091-kde/.thumbs/zypper_up-3.png');
File::Path::make_path('t/data/openqa/testresults/00099937-opensuse-13.1-DVD-i586-Build0091-kde/.thumbs/');
symlink('../../../../images/34/.thumbs/7da661d0c3faf37d49d33b6fc308f2.png',
't/data/openqa/testresults/00099/00099937-opensuse-13.1-DVD-i586-Build0091-kde/.thumbs/zypper_up-3.png');
like(
readlink(
't/data/openqa/testresults/00099/00099937-opensuse-13.1-DVD-i586-Build0091-kde/.thumbs/zypper_up-3.png'),
qr{\Q/34/.thumbs/7da661d0c3faf37d49d33b6fc308f2.png\E},
'link correct'
);

run_gru('relink_testresults' => {max_job => 1000000, min_job => 0});
like(
readlink(
't/data/openqa/testresults/00099/00099937-opensuse-13.1-DVD-i586-Build0091-kde/.thumbs/zypper_up-3.png'),
qr{\Qimages/347/da6/.thumbs/61d0c3faf37d49d33b6fc308f2.png\E},
'relinked'
);
};

subtest 'rm_compat_symlinks' => sub {
File::Path::make_path(join('/', $OpenQA::Utils::imagesdir, '34', '.thumbs'));
symlink(
'../../../images/347/da6/.thumbs/61d0c3faf37d49d33b6fc308f2.png',
't/data/openqa/images/34/.thumbs/7da661d0c3faf37d49d33b6fc308f2.png'
);

ok(-e 't/data/openqa/images/34/.thumbs/7da661d0c3faf37d49d33b6fc308f2.png', 'thumb is there');
run_gru('rm_compat_symlinks' => {});
ok(!-e 't/data/openqa/images/34/.thumbs/7da661d0c3faf37d49d33b6fc308f2.png', 'thumb is gone');
};

subtest 'human readable size' => sub {
is(human_readable_size(13443399680), '13GiB', 'two digits GB');
is(human_readable_size(8007188480), '7.5GiB', 'smaller GB');
Expand Down