enqueue a job to remove old assets whenever a new one is scheduled

os-autoinst · Apr 5, 2015 · 7a736a9 · 7a736a9
1 parent 2009328
commit 7a736a9
Show file tree

Hide file tree

Showing 7 changed files with 57 additions and 7 deletions.
diff --git a/lib/OpenQA.pm b/lib/OpenQA.pm
@@ -461,6 +461,7 @@ sub startup {
 
     $self->gru->add_task(optipng => \&OpenQA::Schema::Result::Jobs::optipng);
     $self->gru->add_task(reduce_result => \&OpenQA::Schema::Result::Jobs::reduce_result);
+    $self->gru->add_task(limit_assets => \&OpenQA::Schema::Result::Assets::limit_assets);
 
     # start workers checker
     $self->_workers_checker;

diff --git a/lib/OpenQA/Plugin/Gru.pm b/lib/OpenQA/Plugin/Gru.pm
@@ -89,14 +89,14 @@ sub run_first {
     my ($self) = @_;
 
     my $dtf = $self->app->schema->storage->datetime_parser;
-    my $where = { run_at => { '<',$dtf->format_datetime(DBIx::Class::Timestamps::now()) } };
+    my $where = { run_at => { '<=',$dtf->format_datetime(DBIx::Class::Timestamps::now()) } };
     my $first = $self->app->schema->resultset('GruTasks')->search($where,{ order_by => qw/id/ })->first;
 
     if ($first) {
         $self->app->log->debug(sprintf("Running Gru task %d(%s)", $first->id, $first->taskname));
         my $subref = $self->app->gru->{tasks}->{$first->taskname};
         if ($subref) {
-            eval { &$subref($first->args) };
+            eval { &$subref($self->app, $first->args) };
             if ($@) {
                 print $@ . "\n";
                 return;
@@ -109,9 +109,10 @@ sub run_first {
 
 sub cmd_run {
     my $self = shift;
+    my $opt = $_[0] || '';
     while (1) {
         if (!$self->run_first) {
-            if ($_[0] eq '-o') {
+            if ($opt eq '-o') {
                 return;
             }
             sleep(5);

diff --git a/lib/OpenQA/Scheduler.pm b/lib/OpenQA/Scheduler.pm
@@ -218,6 +218,7 @@ sub job_create {
     }
 
     my $job = schema->resultset("Jobs")->create(\%new_job_args);
+    $OpenQA::Utils::app->gru->enqueue('limit_assets') if $OpenQA::Utils::app;
 
     job_notify_workers() unless $no_notify;
     return $job;

diff --git a/lib/OpenQA/Schema/Result/Assets.pm b/lib/OpenQA/Schema/Result/Assets.pm
@@ -41,4 +41,46 @@ __PACKAGE__->add_unique_constraint([qw/type name/]);
 __PACKAGE__->has_many(jobs_assets => 'OpenQA::Schema::Result::JobsAssets', 'asset_id');
 __PACKAGE__->many_to_many(jobs => 'jobs_assets', 'job');
 
+sub limit_assets {
+    my ($app) = @_;
+    my $groups = $app->db->resultset('JobGroups')->search({},{ select => 'id' });
+    my %keep;
+    while (my $g = $groups->next) {
+        my $sizelimit = 20 * 1024 * 1024 * 1024; # 20GB
+        my $assets = $app->db->resultset('JobsAssets')->search(
+            {
+                job_id => { -in => $g->jobs->get_column('id')->as_query },
+                'asset.type' => ['iso']
+            },
+            {
+                join => 'asset',
+                order_by => 'asset.t_created desc',
+                select => [qw/asset.name asset.type asset.id/],
+                as => [qw/asset_name asset_type asset_id/],
+                distinct => 1
+            }
+        );
+        while (my $a = $assets->next) {
+            next if $a->get_column('asset_type') ne 'iso';
+            my $file = sprintf("%s/%s/%s", $OpenQA::Utils::assetdir,$a->get_column('asset_type'),$a->get_column('asset_name'));
+            my @st = stat($file);
+            if (@st && $st[7] > 0) {
+                $keep{$a->asset_id} = 1;
+                $sizelimit -= $st[7];
+            }
+            # check after keeping - so we can be sure to keep at least one even if sizelimit too strict
+            last if $sizelimit < 0;
+        }
+    }
+    if (%keep) {
+        my $assets = $app->db->resultset('Assets')->search({ id => { not_in => [ sort keys %keep ] }});
+        while (my $a = $assets->next) {
+            my $file = sprintf("%s/%s/%s", $OpenQA::Utils::assetdir,$a->type,$a->name);
+            print "RM $file\n";
+            unlink($file);
+            $a->delete;
+        }
+    }
+}
+
 1;
diff --git a/lib/OpenQA/Schema/Result/GruTasks.pm b/lib/OpenQA/Schema/Result/GruTasks.pm
@@ -56,7 +56,7 @@ __PACKAGE__->filter_column(
 
 sub decode_json {
     my $ret = JSON::decode_json($_[1]);
-    return $ret->{_} if defined $ret->{_};
+    return $ret->{_} if ref($ret) eq 'HASH' && defined $ret->{_};
     return $ret;
 }
 

diff --git a/lib/OpenQA/Schema/Result/Jobs.pm b/lib/OpenQA/Schema/Result/Jobs.pm
@@ -479,7 +479,7 @@ sub insert_test_modules($) {
 
 # gru job
 sub reduce_result {
-    my ($resultdir) = @_;
+    my ($app, $resultdir) = @_;
 
     $resultdir .= "/";
     unlink($resultdir . "autoinst-log.txt");
@@ -557,8 +557,9 @@ sub running_modinfo() {
 }
 
 sub optipng {
-    OpenQA::Utils::log_debug("optipng $_[0]");
-    system("optipng", "-preserve", "-o2", $_[0]);
+    my ($app, $path) = @_;
+    OpenQA::Utils::log_debug("optipng $path");
+    system("optipng", "-preserve", "-o2", $path);
 }
 
 sub store_image {

diff --git a/t/14-grutasks.t b/t/14-grutasks.t
@@ -47,4 +47,8 @@ like($output, qr,optipng .*'$file';,, 'optipng queued');
 $c->run('run', '-o');
 is((stat($file))[7], 286, 'optimized file size');
 
+# now to something completely different
+$t->app->gru->enqueue('limit_assets');
+$c->run('run', '-o');
+
 done_testing();