Merge pull request #7883 from mlschroe/master

Support generated build requires

src/api/test/test_helper.rb

@@ -104,7 +104,7 @@ def inject_build_job(project, package, repo, arch, extrabinary = nil)
   f.close
   extrabinary = " -o -name #{extrabinary}" if extrabinary
   # rubocop:disable Metrics/LineLength
-  system("cd #{Rails.root}/test/fixtures/backend/binary/; exec find . -name '*#{arch}.rpm' -o -name '*src.rpm' -o -name logfile -o -name _statistics #{extrabinary} | cpio -H newc -o 2>/dev/null | curl -s -X POST -T - 'http://localhost:3201/putjob?arch=#{arch}&code=success&job=#{jobfile.gsub(/.*\//, '')}&jobid=#{jobid}' > /dev/null")
+  system("cd #{Rails.root}/test/fixtures/backend/binary/; exec find . -name '*#{arch}.rpm' -o -name '*src.rpm' -o -name logfile -o -name _statistics #{extrabinary} | cpio -H newc -o 2>/dev/null | curl -s -X POST -T - 'http://localhost:3201/putjob?arch=#{arch}&code=succeeded&job=#{jobfile.gsub(/.*\//, '')}&jobid=#{jobid}' > /dev/null")
   # rubocop:enable Metrics/LineLength
   system("echo \"#{verifymd5}  #{package}\" > #{jobfile}:dir/meta")
 end

src/backend/BSSched/BuildJob.pm

@@ -368,6 +368,30 @@ sub update_buildavg {
   $gctx->{'buildavg'} = $buildavg;
 }
 
+sub set_genbuildreqs {
+  my ($gctx, $prp, $packid, $file) = @_;
+  my $filecontent = $file ? readstr($file, 1) : undef;
+  my $genbuildreqs = $gctx->{'genbuildreqs'}->{$prp};
+  if (defined $filecontent) {
+    my $md5 = Digest::MD5::md5_hex($filecontent);
+    return if $genbuildreqs && ($genbuildreqs->{$packid} || [''])->[0] eq $md5;
+    $genbuildreqs = $gctx->{'genbuildreqs'}->{$prp} = {} unless $genbuildreqs;
+    $genbuildreqs->{$packid} = [ $md5, [ split("\n", $filecontent) ] ];
+  } else {
+    return if !$genbuildreqs || !delete($genbuildreqs->{$packid});
+    delete($gctx->{'genbuildreqs'}->{$prp}) if !%$genbuildreqs;
+  }
+  my $myarch = $gctx->{'arch'};
+  my $reporoot = $gctx->{'reporoot'};
+  my $gdst = "$reporoot/$prp/$myarch";
+  if (%{$genbuildreqs || {}}) {
+    mkdir_p($gdst);
+    BSUtil::store("$gdst/.:genbuildreqs", "$gdst/:genbuildreqs", $genbuildreqs);
+  } else {
+    unlink("$gdst/:genbuildreqs");
+  }
+}
+
 =head2 jobfinished - called when a build job is finished
 
  - move artifacts into built result dir
@@ -440,7 +464,7 @@ sub jobfinished {
   delete $status->{'uri'};      # obsolete
 
   my $code = $js->{'result'};
-  $code = 'failed' unless $code eq 'succeeded' || $code eq 'unchanged';
+  $code = 'failed' unless $code eq 'succeeded' || $code eq 'unchanged' || $code eq 'genbuildreqs';
 
   my @all = ls($jobdatadir);
   my %all = map {$_ => 1} @all;
@@ -505,6 +529,14 @@ sub jobfinished {
     $changed->{$prp} ||= 1;     # package is no longer blocking
     return;
   }
+  if ($code eq 'genbuildreqs') {
+    print "  - $job: build has different generated build requires\n";
+    set_genbuildreqs($gctx, $prp, $packid, "$jobdatadir/_generated_buildreqs");
+    unlink($_) for @all;
+    rmdir($jobdatadir);
+    $changed->{$prp} ||= 1;     # package is no longer blocking
+    return;
+  }
   print "  - $prp: $packid built: ".(@all). " files\n";
   mkdir_p("$gdst/:logfiles.success");
   mkdir_p("$gdst/:logfiles.fail");
@@ -926,12 +958,14 @@ sub create {
     $ctx->{"sysbuild_$buildtype"} ||= [ Build::get_sysbuild($bconf, $buildtype) ];
     @sysdeps = @{$ctx->{"sysbuild_$buildtype"}};
   }
-  add_expanddebug($ctx,'sysdeps expansion') if $expanddebug && @sysdeps;
+  add_expanddebug($ctx, 'sysdeps expansion', undef, $ctx->{'pool'}) if $expanddebug && @sysdeps;
   @btdeps = () if @sysdeps;	# already included in sysdeps
 
   # calculate packages needed for building
+  my $genbuildreqs = ($ctx->{'genbuildreqs'} || {})->{$packid};
   my @bdeps = grep {!/^\// || $bconf->{'fileprovides'}->{$_}} @{$info->{'prereq'} || []};
   unshift @bdeps, '--directdepsend--' if @bdeps;
+  unshift @bdeps, @{$genbuildreqs->[1]} if $genbuildreqs;
   unshift @bdeps, @{$info->{'dep'} || []}, @btdeps, @{$ctx->{'extradeps'} || []};
   push @bdeps, '--ignoreignore--' if @sysdeps;
 
@@ -1057,7 +1091,9 @@ sub create {
     }
   }
   if (!$ctx->{'isreposerver'}) {
+    my $genbuildreqs = ($ctx->{'genbuildreqs'} || {})->{$packid};
     $binfo->{'logidlelimit'} = $bconf->{'buildflags:logidlelimit'} if $bconf->{'buildflags:logidlelimit'};
+    $binfo->{'genbuildreqs'} = $genbuildreqs->[0] if $genbuildreqs;
   }
   $ctx->writejob($job, $binfo, $reason);
 

src/backend/BSSched/BuildJob/Import.pm

@@ -115,16 +115,15 @@ sub createexportjob {
 =cut
 
 sub jobfinished {
-  my ($ectx, $job, $js) = @_;
+  my ($ectx, $job, $info, $js) = @_;
 
   my $gctx = $ectx->{'gctx'};
   my $changed = $gctx->{'changed_med'};
   my $myjobsdir = $gctx->{'myjobsdir'};
   my $myarch = $gctx->{'arch'};
-  my $info = readxml("$myjobsdir/$job", $BSXML::buildinfo, 1);
   my $jobdatadir = "$myjobsdir/$job:dir";
-  if (!$info || ! -d $jobdatadir) {
-    print "  - $job is bad\n";
+  if (! -d $jobdatadir) {
+    print "  - $job has no data dir\n";
     return;
   }
   my $projid = $info->{'project'};

src/backend/BSSched/BuildJob/Upload.pm

@@ -36,20 +36,15 @@ BSSched::BuildJob::Upload - A Class to handle upload jobs
 =cut
 
 sub jobfinished {
-  my ($ectx, $job, $js) = @_;
+  my ($ectx, $job, $info, $js) = @_;
 
   my $gctx = $ectx->{'gctx'};
   my $myarch = $gctx->{'arch'};
   my $changed = $gctx->{'changed_med'};
   my $myjobsdir = $gctx->{'myjobsdir'};
-  my $info = readxml("$myjobsdir/$job", $BSXML::buildinfo, 1);
   my $jobdatadir = "$myjobsdir/$job:dir";
-  if (!$info || ! -d $jobdatadir) {
-    print "  - $job is bad\n";
-    return;
-  }
-  if ($info->{'arch'} ne $myarch) {
-    print "  - $job has bad arch\n";
+  if (! -d $jobdatadir) {
+    print "  - $job has no data dir\n";
     return;
   }
   my $projid = $info->{'project'};

src/backend/BSSched/Checker.pm

@@ -313,6 +313,17 @@ sub setup {
     }
     $ctx->{'onlybuild'} = \%onlybuild if %onlybuild;
   }
+
+  # sync genbuildreqs from on-disk version
+  my $genbuildreqs = {};
+  $genbuildreqs = BSUtil::retrieve("$gdst/:genbuildreqs", 1) || {} if -e "$gdst/:genbuildreqs";
+  $ctx->{'genbuildreqs'} = $genbuildreqs;
+  if (%$genbuildreqs) {
+    $gctx->{'genbuildreqs'}->{$prp} = $genbuildreqs;
+  } else {
+    delete $gctx->{'genbuildreqs'}->{$prp} ;
+  }
+
   return ('scheduling', undef);
 }
 
@@ -507,6 +518,7 @@ sub expandandsort {
 
   $ctx->{'experrors'} = \%experrors;
   my $packs = $ctx->{'packs'};
+  my $genbuildreqs_prp = $ctx->{'genbuildreqs'} || {};
   for my $packid (@$packs) {
     my $pdata = $pdatas->{$packid};
 
@@ -574,6 +586,7 @@ sub expandandsort {
       next;
     }
     my @deps = @{$info->{'dep'} || []};
+    push @deps, @{$genbuildreqs_prp->{$packid}->[1]} if $genbuildreqs_prp->{$packid};
     my $handler = $handlers{$buildtype} || $handlers{default};
     my ($eok, @edeps) = $handler->expand($bconf, $subpacks->{$info->{'name'}}, @deps);
     if (!$eok) {

src/backend/BSSched/EventHandler.pm

@@ -397,6 +397,7 @@ sub event_wipe {
   print "wiping $prp $packid\n";
   my $prpsearchpath = $gctx->{'prpsearchpath'}->{$prp};
   BSSched::BuildResult::wipe($gctx, $prp, $packid, $prpsearchpath, $ectx->{'dstcache'}) if -d "$gdst/$packid";
+  BSSched::BuildJob::set_genbuildreqs($gctx, $prp, $packid, undef);
   for $prp (@{$gctx->{'prps'}}) {
     if ((split('/', $prp, 2))[0] eq $projid) {
       $changed_high->{$prp} = 2;

src/backend/BSXML.pm

@@ -544,6 +544,7 @@ our $buildinfo = [
 	'genmetaalgo',	# internal
 	'logidlelimit',	# internal
 	'logsizelimit',	# internal
+	'genbuildreqs',	# internal
       [ 'dep' ],
      [[ 'bdep' =>
 	'name',

src/backend/bs_publish

@@ -972,6 +972,9 @@ EOL
     push @signargs, @{$data->{'signargs'} || []};
     qsystem($BSConfig::sign, @signargs, '-d', "$extrep/Release") && die("    sign failed: $?\n");
     rename("$extrep/Release.asc","$extrep/Release.gpg");
+    BSUtil::cp("$extrep/Release", undef, "$extrep/InRelease.tmp");
+    qsystem($BSConfig::sign, @signargs, '-c', "$extrep/InRelease.tmp") && die("    sign failed: $?\n");
+    rename("$extrep/InRelease.tmp","$extrep/InRelease");
   }
   if ($BSConfig::sign) {
     writestr("$extrep/Release.key", undef, $data->{'pubkey'}) if $data->{'pubkey'};

src/backend/bs_repserver

@@ -2247,18 +2247,17 @@ sub putjob {
   }
   $jobstatus->{'code'} = 'finished';
   $jobstatus->{'endtime'} = $now;
-  $jobstatus->{'result'} = 'failed';
-  # upload is empty for local image building
-  if (!@$uploaded) {
-    $jobstatus->{'result'} = $cgi->{'code'} || 'succeeded';
-  }
-  # usual build should have uploaded content.
-  for my $file (@$uploaded) {
-    next if $file->{'name'} eq 'meta' || $file->{'name'} eq 'logfile';
-    $jobstatus->{'result'} = 'succeeded';
-    last;
-  }
-  $jobstatus->{'result'} = 'unchanged' if $cgi->{'code'} && $cgi->{'code'} eq 'unchanged';
+  my $code = $cgi->{'code'};
+  # compat: calculate code from upload list if not provided
+  if (!$code) {
+    $code = 'failed';
+    # upload is empty for local image building
+    $code = 'succeeded' if !@$uploaded;
+    # successful builds have uploaded content.
+    $code = 'succeeded' if grep {$_->{'name'} ne 'meta' && $_->{'name'} ne 'logfile'} @$uploaded;
+  }
+  $code = 'failed' if $code ne 'succeeded' && $code ne 'failed' && $code ne 'unchanged' && $code ne 'genbuildreqs';
+  $jobstatus->{'result'} = $code;
 
   notify_jobresult($info, $jobstatus, "$projid/$repoid/$arch");
 

src/backend/bs_sched

@@ -437,6 +437,7 @@ my $gctx = {
   'remotemissing' => {},		# missing remote projects cache
   'remoteprojs_changed' => {},		# entries changed in remoteprojs hash
   'projsuspended' => {},		# project is suspended for now
+  'genbuildreqs' => {},			# generated build require dependencies
 
   # lastcheck cache
   'lastcheck' => {},			# package check data of last check
@@ -955,6 +956,7 @@ eval {
 	$gctx->{'repodatas'}->drop($prp, $myarch);
         delete $gctx->{'lastcheck'}->{$prp};
         delete $gctx->{'prpcheckuseforbuild'}->{$prp};
+        delete $gctx->{'genbuildreqs'}->{$prp};
         my $ctx = BSSched::Checker->new($gctx, $prp);
         $ctx->wipeobsoleterepo();
       }

src/backend/bs_worker

@@ -3171,8 +3171,17 @@ sub dobuild {
     } else {
       return 2;
     }
-  } elsif ($ret == 768) {
+  } elsif ($ret == 3 * 256) {
     return 3;
+  } elsif ($ret == 9 * 256) {
+    if ($vm =~ /(xen|kvm|zvm|emulator|pvm|openstack)/) {
+      rm_rf("$buildroot/.build.packages");
+      if (!rename("$buildroot/.mount/.build.packages", "$buildroot/.build.packages")) {
+        print "final rename failed: $!\n";
+        return 1;
+      }
+    }
+    return -e "$buildroot/.build.packages/OTHER/_statistics" ? 9 : 1;
   }
   if ($ret) {
     return 1;
@@ -3194,7 +3203,7 @@ sub dobuild {
     symlink('.', "$buildroot/.build.packages/DEBS");
     symlink('.', "$buildroot/.build.packages/KIWI");
     # convert build statistics into xml
-    if( -e "$buildroot/.build.packages/OTHER/_statistics") {
+    if (-e "$buildroot/.build.packages/OTHER/_statistics") {
       my $iosectors = 0;
       my $iorequests = 0;
       open(FILE, "<", "$buildroot/.build.packages/OTHER/_statistics") || die;
@@ -3832,6 +3841,22 @@ if ($hostcheck) {
   }
 }
 
+my $send_genbuildreqs;
+if ($ex != 3 && -e "$buildroot/.build.packages/OTHER/_generated_buildreqs") {
+  if (-s "$buildroot/.build.packages/OTHER/_generated_buildreqs" > 100000) {
+    $ex = 1;
+  } else {
+    my $genbuildreqs = readstr("$buildroot/.build.packages/OTHER/_generated_buildreqs");
+    $send_genbuildreqs = { name => '_generated_buildreqs', filename => "$buildroot/.build.packages/OTHER/_generated_buildreqs" };
+    if ($genbuildreqs =~ /[^\n\040-\176]/s) {
+      $ex = 1;
+      $send_genbuildreqs = undef;
+    } elsif (Digest::MD5::md5_hex($genbuildreqs) ne ($buildinfo->{'genbuildreqs'} || '')) {
+      $ex = 11;
+    }
+  }
+}
+
 my @send;
 my $kiwitree;
 if ($ex == 0 && $buildinfo->{'reason'} ne "rebuild counter sync" && -f "$buildroot/.build.packages/same_result_marker") {
@@ -3877,6 +3902,7 @@ if ($ex == 0) {
     push @send, map {"$buildroot/.build.packages/$d/$_"} @files;
   }
   @send = map {{name => (split('/', $_))[-1], filename => $_}} @send;
+  @send = grep {$_->{'name'} ne '_generated_buildreqs'} @send;
   if ($kiwitree) {
     my $kiwitreefile = "$buildroot/.build.packages/.kiwitree";
     eval {
@@ -3926,6 +3952,9 @@ if (!$ex) {
       $code = 'failed';
     }
   }
+} elsif ($ex == 11) {
+  print "build has different generated build requires\n";
+  $code = 'genbuildreqs';
 } else {
   print "build failed, send back logfile...\n";
   $code = 'failed';
@@ -3943,6 +3972,7 @@ if ($code eq 'badhost') {
     push @send, {name => 'logfile', data => "no build log?\n"};
   }
 }
+push @send, $send_genbuildreqs if $send_genbuildreqs && $ex != 2 && $ex != 3;
 
 if (!$testmode) {
   my $param = {