Merge pull request #1938 from kraih/clean_gru

Actually use a Minion worker for Gru

lib/OpenQA/WebAPI.pm

@@ -72,6 +72,9 @@ sub startup {
     # set cookie timeout to 48 hours (will be updated on each request)
     $self->app->sessions->default_expiration(48 * 60 * 60);
 
+    # commands
+    push @{$self->commands->namespaces}, 'OpenQA::WebAPI::Command';
+
     # add actions before dispatching page
     $self->hook(
         before_dispatch => sub {

lib/OpenQA/WebAPI/Command/gru.pm

@@ -0,0 +1,45 @@
+# Copyright (C) 2018 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package OpenQA::WebAPI::Command::gru;
+use Mojo::Base 'Mojolicious::Commands';
+
+has description => 'Gru job queue';
+has hint        => <<EOF;
+See 'APPLICATION gru help COMMAND' for more information on a specific
+command.
+EOF
+has message    => sub { shift->extract_usage . "\nCommands:\n" };
+has namespaces => sub { ['OpenQA::WebAPI::Command::gru'] };
+
+sub help { shift->run(@_) }
+
+1;
+
+=encoding utf8
+
+=head1 NAME
+
+OpenQA::WebAPI::Command::gru - gru command
+
+=head1 SYNOPSIS
+
+  Usage: APPLICATION gru COMMAND [OPTIONS]
+
+=head1 DESCRIPTION
+
+L<OpenQA::WebAPI::Command::gru> lists available Gru commands.
+
+=cut

lib/OpenQA/WebAPI/Command/gru/list.pm

@@ -0,0 +1,45 @@
+# Copyright (C) 2018 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package OpenQA::WebAPI::Command::gru::list;
+use Mojo::Base 'Minion::Command::minion::job';
+
+has description => 'List Gru jobs and more';
+has usage       => sub { shift->extract_usage };
+
+1;
+
+=encoding utf8
+
+=head1 NAME
+
+OpenQA::WebAPI::Command::gru::list - Gru list command
+
+=head1 SYNOPSIS
+
+  Usage: APPLICATION gru list [OPTIONS] [IDS]
+
+    script/openqa gru list
+
+  Options:
+    See 'script/openqa minion job -h' for all available options.
+
+=head1 DESCRIPTION
+
+L<OpenQA::WebAPI::Command::gru::list> is a subclass of
+L<Minion::Command::minion::job> that merely renames the command for backwards
+compatibility.
+
+=cut

lib/OpenQA/WebAPI/Command/gru/run.pm

@@ -0,0 +1,79 @@
+# Copyright (C) 2018 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package OpenQA::WebAPI::Command::gru::run;
+use Mojo::Base 'Minion::Command::minion::worker';
+
+use Mojo::Util 'getopt';
+use OpenQA::WebAPI::GruJob;
+
+has description => 'Start Gru worker';
+has usage       => sub { shift->extract_usage };
+
+sub run {
+    my ($self, @args) = @_;
+
+    getopt \@args, 'o|oneshot' => \(my $oneshot);
+
+    my $minion = $self->app->minion;
+    $minion->on(
+        worker => sub {
+            my ($minion, $worker) = @_;
+
+            # Only one job can run at a time for now (until all Gru tasks are parallelism safe)
+            $worker->status->{jobs} = 1;
+
+            $worker->on(
+                dequeue => sub {
+                    my ($worker, $job) = @_;
+
+                    # Reblessing the job is fine for now, but in the future it would be nice
+                    # to use a role instead
+                    bless $job, 'OpenQA::WebAPI::GruJob';
+                });
+        });
+
+    if   ($oneshot) { $minion->perform_jobs }
+    else            { $self->SUPER::run(@args) }
+}
+
+1;
+
+=encoding utf8
+
+=head1 NAME
+
+OpenQA::WebAPI::Command::gru::run - Gru run command
+
+=head1 SYNOPSIS
+
+  Usage: APPLICATION gru run [OPTIONS]
+
+    script/openqa gru run
+    script/openqa gru run -o
+
+  Options:
+    -o, --oneshot   Perform all currently enqueued jobs and then exit
+
+    See 'script/openqa minion worker -h' for all available options.
+
+
+=head1 DESCRIPTION
+
+L<OpenQA::WebAPI::Command::gru::run> is a subclass of
+L<Minion::Command::minion::worker> that adds Gru features with
+L<OpenQA::WebAPI::GruJob>.
+
+=cut

lib/OpenQA/WebAPI/GruJob.pm

@@ -0,0 +1,88 @@
+# Copyright (C) 2018 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package OpenQA::WebAPI::GruJob;
+use Mojo::Base 'Minion::Job';
+
+use OpenQA::Utils 'log_error';
+
+sub execute {
+    my $self = shift;
+
+    my $ttl     = $self->info->{notes}{ttl};
+    my $elapsed = time - $self->info->{created};
+    if (defined $ttl && $elapsed > $ttl) {
+        my $ttl_error = 'TTL Expired';
+        if (exists $self->info->{notes}{gru_id}) {
+            $self->_fail_gru($self->info->{notes}{gru_id} => $ttl_error) if $self->fail({error => $ttl_error});
+        }
+        else {
+            $self->fail({error => $ttl_error});
+        }
+        return undef;
+    }
+
+    my ($buffer, $err);
+    {
+        open my $handle, '>', \$buffer;
+        local *STDERR = $handle;
+        local *STDOUT = $handle;
+        $err = $self->SUPER::execute;
+    };
+
+    if (defined $err) {
+        log_error("Gru command issue: $err");
+        $self->_fail_gru($self->info->{notes}{gru_id} => $err)
+          if $self->fail({(output => $buffer) x !!(defined $buffer), error => $err})
+          && exists $self->info->{notes}{gru_id};
+    }
+    else {
+        $self->finish(defined $buffer ? $buffer : 'Job successfully executed');
+        $self->_delete_gru($self->info->{notes}{gru_id}) if exists $self->info->{notes}{gru_id};
+    }
+
+    return undef;
+}
+
+sub _delete_gru {
+    my ($self, $id) = @_;
+    my $gru = $self->minion->app->db->resultset('GruTasks')->find($id);
+    $gru->delete() if $gru;
+}
+
+sub _fail_gru {
+    my ($self, $id, $reason) = @_;
+    my $gru = $self->minion->app->db->resultset('GruTasks')->find($id);
+    $gru->fail($reason) if $gru;
+}
+
+1;
+
+=encoding utf8
+
+=head1 NAME
+
+OpenQA::WebAPI::GruJob - A Gru Job
+
+=head1 SYNOPSIS
+
+    use OpenQA::WebAPI::GruJob;
+
+=head1 DESCRIPTION
+
+L<OpenQA::WebAPI::GruJob> is a subclass of L<Minion::Job> used by
+L<OpenQA::WebAPI::Plugin::Gru> that adds Gru metadata handling and TTL support.
+
+=cut