Skip to content

Commit

Permalink
Merge pull request #13981 from adrianschroeter/product_speedup
Browse files Browse the repository at this point in the history 
Product speedup
  • Loading branch information
@mlschroe
mlschroe committed Mar 13, 2023
2 parents 9ae17f3 + 38907fb commit fa5742d
Show file tree
Hide file tree
Showing 8 changed files with 167 additions and 103 deletions.
29 changes: 19 additions & 10 deletions src/backend/BSRepServer.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use strict;
use BSConfiguration;
use BSRPC ':https';
use BSUtil;
use BSVerify;
use Build;

my $reporoot = "$BSConfig::bsdir/build";
Expand Down Expand Up @@ -59,7 +60,7 @@ sub addrepo_scan {
}

sub read_gbininfo {
my ($dir, $collect) = @_;
my ($dir, $collect, $withquery) = @_;
my $gbininfo = BSUtil::retrieve("$dir/:bininfo", 1);
if ($gbininfo) {
my $gbininfo_m = BSUtil::retrieve("$dir/:bininfo.merge", 1);
Expand All @@ -79,27 +80,35 @@ sub read_gbininfo {
$gbininfo = {};
for my $packid (grep {!/^[:\.]/} sort(ls($dir))) {
next if $packid eq '_deltas' || $packid eq '_volatile' || ! -d "$dir/$packid";
$gbininfo->{$packid} = read_bininfo("$dir/$packid");
$gbininfo->{$packid} = read_bininfo("$dir/$packid", $withquery);
}
}
return $gbininfo;
}

sub read_bininfo {
my ($dir) = @_;
my ($dir, $withquery) = @_;
my $bininfo = BSUtil::retrieve("$dir/.bininfo", 1);
return $bininfo if $bininfo;
# .bininfo not present or old style, generate it
$bininfo = {};
for my $file (ls($dir)) {
if ($file =~ /\.(?:$binsufsre)$/) {
my @s = stat("$dir/$file");
my ($hdrmd5, $leadsigmd5);
$hdrmd5 = Build::queryhdrmd5("$dir/$file", \$leadsigmd5);
next unless $hdrmd5;
my $r = {'filename' => $file, 'id' => "$s[9]/$s[7]/$s[1]"};
$r->{'hdrmd5'} = $hdrmd5;
$r->{'leadsigmd5'} = $leadsigmd5 if $leadsigmd5;
my $r = {};
eval {
my $leadsigmd5;
$r->{'hdrmd5'} = Build::queryhdrmd5("$dir/$file", \$leadsigmd5);
if ($withquery) {
$r = Build::query("$dir/$file", 'evra' => 1);
BSVerify::verify_nevraquery($r) if $r;
}
die("missing hdrmd5\n") unless $r && $r->{'hdrmd5'};
$r->{'leadsigmd5'} = $leadsigmd5 if $leadsigmd5;
};
next if $@;
$r->{'filename'} = $file;
$r->{'id'} = "$s[9]/$s[7]/$s[1]";
$bininfo->{$file} = $r;
} elsif ($file =~ /\.obsbinlnk$/) {
my @s = stat("$dir/$file");
Expand All @@ -108,7 +117,7 @@ sub read_bininfo {
my $r = {%$d, 'filename' => $file, 'id' => "$s[9]/$s[7]/$s[1]"};
delete $r->{'path'};
$bininfo->{$file} = $r;
} elsif ($file =~ /[-.]appdata\.xml$/ || $file eq '_modulemd.yaml') {
} elsif ($file =~ /[-.]appdata\.xml$/ || $file eq '_modulemd.yaml' || $file =~ /slsa_provenance\.json$/) {
local *F;
open(F, '<', "$dir/$file") || next;
my @s = stat(F);
Expand Down
134 changes: 79 additions & 55 deletions src/backend/BSSched/BuildJob/KiwiProduct.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,15 @@ sub check {
return ('excluded', 'no architectures for packages') unless grep {$imagearch{$_}} @{$repo->{'arch'} || []};
my @archs = grep {$imagearch{$_}} @{$repo->{'arch'} || []};

# sort archs like in bs_worker
@archs = sort(@archs);
if ($packid =~ /-([^-]+)$/) {
if (grep {$_ eq $1} @archs) {
@archs = grep {$_ ne $1} @archs;
push @archs, $1;
}
}

if ($myarch ne $buildarch && $myarch ne $localbuildarch) {
if (!grep {$_ eq $myarch} @archs) {
if ($ctx->{'verbose'}) {
Expand Down Expand Up @@ -267,7 +276,10 @@ sub check {
my %blockedarch;
my $delayed_errors = '';
my $projpacks = $gctx->{'projpacks'};
my %unneeded_na;
my %archs = map {$_ => 1} @archs;
for my $aprp (@aprps) {
my %seen_fn; # resolve file conflicts in this prp
my %known;
my ($aprojid, $arepoid) = split('/', $aprp, 2);
my $aproj = $projpacks->{$aprojid} || {};
Expand All @@ -279,7 +291,9 @@ sub check {
my $info = (grep {$_->{'repository'} eq $arepoid} @{$pdatas->{$apackid}->{'info'} || []})[0];
$known{$apackid} = $info->{'name'} if $info && $info->{'name'};
}
for my $arch (@archs) {
my $is_maintenance_release = ($aproj->{'kind'} || '') eq 'maintenance_release' ? 1 : 0;
my @next_unneeded_na;
for my $arch (reverse @archs) {
next if $myarch ne $buildarch && $myarch ne $arch;
my $ps;
if (!$remoteprojs->{$aprojid} && -e "$reporoot/$aprp/$arch/:packstatus") {
Expand Down Expand Up @@ -316,96 +330,106 @@ sub check {
# just for maintenance_release project handling, in that case we
# use the binary from the container with the highest number if
# some containers contain the same binary.
my $seen_binary;
$seen_binary = {} if ($aproj->{'kind'} || '') eq 'maintenance_release';
my $seen_binary = $is_maintenance_release ? {} : undef;

for my $apackid (BSSched::ProjPacks::orderpackids($aproj, @apackids)) {
next if $apackid eq '_volatile';
next if ($pdatas->{$apackid} || {})->{'patchinfo'};
my $code = $ps->{$apackid} || 'unknown';

my $apackid2 = $known{$apackid} || $apackid;
if (($allpacks && !$deps{"-$apackid"} && !$deps{"-$apackid2"}) || $deps{$apackid} || $deps{$apackid2}) {
# hey, we probably need this package! wait till it's finished
my $code = $ps->{$apackid} || 'unknown';
if (!$neverblock && ($code eq 'scheduled' || $code eq 'blocked' || $code eq 'finished')) {
# fast blocked check
if (!$neverblock && ($code eq 'scheduled' || $code eq 'blocked' || $code eq 'finished')) {
my $apackid2 = $known{$apackid} || $apackid;
# crude check if the "main" binary is needed
if (($allpacks && !$deps{"-$apackid"} && !$deps{"-$apackid2"}) || $deps{$apackid} || $deps{$apackid2}) {
# hey, we probably need this package! wait till it's finished
push @blocked, "$aprp/$arch/$apackid";
$blockedarch{$arch} = 1;
last if @blocked > $maxblocked;
next;
}
}

# hmm, we don't know if we really need it. check bininfo.
my $bininfo;
if ($gbininfo) {
$bininfo = $gbininfo->{$apackid} || {};
} else {
$bininfo = read_bininfo_oldok("$reporoot/$aprp/$arch/$apackid");
}
# go through all the rpms in this package
my $bininfo = $gbininfo ? $gbininfo->{$apackid} : read_bininfo_oldok("$reporoot/$aprp/$arch/$apackid");
next if !$bininfo || $bininfo->{'.nouseforbuild'}; # skip channels/patchinfos

# skip channels/patchinfos
next if $bininfo->{'.nouseforbuild'};
my @bi = sort(keys %$bininfo);

my @got;
# first find out if we need any rpm from this package
my $needit;
my @bi = sort(keys %$bininfo);
for my $fn (@bi) {
next unless $fn =~ /^(?:::import::.*::)?(.+)-(?:[^-]+)-(?:[^-]+)\.([a-zA-Z][^\.\-]*)\.rpm$/;
my ($bn, $ba) = ($1, $2);
next if $ba eq 'src' || $ba eq 'nosrc'; # always unneeded
my $na = "$bn.$ba";
next if $unneeded_na{$na};
if ($fn =~ /-(?:debuginfo|debugsource)-/) {
if ($nodbgpkgs || !$deps{$bn}) {
$unneeded_na{$na} = 1;
next;
}
}
next if $seen_binary && $seen_binary->{$na};
next if $seen_fn{$fn};
if ($fn =~ /^::import::(.*?)::(.*)$/) {
next if $archs{$1}; # we pick it up from the real arch
next if $seen_fn{$2};
}
if (!($deps{$bn} || ($allpacks && !$deps{"-$bn"}))) {
$unneeded_na{$na} = 1; # cache unneeded
next;
}
$needit = 1;
last;
}

next unless $needit;

# put imports last
my @ibi = grep {/^::import::/} @bi;
if (@ibi) {
@bi = grep {!/^::import::/} @bi;
push @bi, @ibi;
}

# we need the package, add all rpms
for my $fn (@bi) {
next unless $fn =~ /\.rpm$/;
next unless $fn =~ /^(?:::import::.*::)?(.+)-(?:[^-]+)-(?:[^-]+)\.([a-zA-Z][^\.\-]*)\.rpm$/;
my ($bn, $ba) = ($1, $2);
next if $nosrcpkgs && ($ba eq 'src' || $ba eq 'nosrc');
next if $nodbgpkgs && $fn =~ /-(?:debuginfo|debugsource)-/;
next if $fn =~ /\.(?:nosrc|src)\.rpm$/;
if ($fn =~ /^::import::.*?::(.*)$/) {
# ignore import if we already got the package (can happen with aggregates)
next if $rpms_meta{"$aprp/$arch/$apackid/$1"};
next if $seen_fn{$fn};
my $na = "$bn.$ba";
next if $seen_binary && $seen_binary->{$na}++;
if ($fn =~ /^::import::(.*?)::(.*)$/) {
next if $archs{$1}; # we pick it up from the real arch
next if $seen_fn{$2};
}
my $b = $bininfo->{$fn};
if ($seen_binary) {
next if $seen_binary->{"$b->{'name'}.$b->{'arch'}"};
$seen_binary->{"$b->{'name'}.$b->{'arch'}"} = 1;
}
$needit = 1 if $deps{$b->{'name'}} || ($allpacks && !$deps{"-$b->{'name'}"});
push @got, "$aprp/$arch/$apackid/$fn";
$rpms_hdrmd5{$got[-1]} = $b->{'hdrmd5'} if $b->{'hdrmd5'};
$rpms_meta{$got[-1]} = "$aprp/$arch/$apackid/$b->{'name'}.$b->{'arch'}";
my $rpm = "$aprp/$arch/$apackid/$fn";
push @rpms, $rpm;
$rpms_hdrmd5{$rpm} = $b->{'hdrmd5'} if $b->{'hdrmd5'};
$rpms_meta{$rpm} = "$aprp/$arch/$apackid/$na";
$seen_fn{$fn} = 1;
push @next_unneeded_na, $na unless $ba eq 'src' || $ba eq 'nosrc';
}
next unless $needit;
# collect all source packages if we needed one binary package
if (!$nosrcpkgs) {
for my $fn (@bi) {
next unless $fn =~ /\.(?:nosrc|src)\.rpm$/;
if ($fn =~ /^::import::.*?::(.*)$/) {
# ignore import if we already got the package (can happen with aggregates)
next if $rpms_meta{"$aprp/$arch/$apackid/$1"};
}
my $b = $bininfo->{$fn};
if ($seen_binary) {
next if $seen_binary->{"$b->{'name'}-$b->{'version'}-$b->{'release'}.$b->{'arch'}"};
$seen_binary->{"$b->{'name'}-$b->{'version'}-$b->{'release'}.$b->{'arch'}"} = 1;
}
push @got, "$aprp/$arch/$apackid/$fn";
$rpms_hdrmd5{$got[-1]} = $b->{'hdrmd5'} if $b->{'hdrmd5'};
$rpms_meta{$got[-1]} = "$aprp/$arch/$apackid/$b->{'name'}.$b->{'arch'}";
}
}
# ok we need it. check if the package is built.
my $code = $ps->{$apackid} || 'unknown';

# check if we are blocked
if (!$neverblock && ($code eq 'scheduled' || $code eq 'blocked' || $code eq 'finished')) {
push @blocked, "$aprp/$arch/$apackid";
$blockedarch{$arch} = 1;
last if @blocked > $maxblocked;
next;
}
push @rpms, @got;
}
last if @blocked > $maxblocked;
}
@next_unneeded_na = () if $deps{'--use-newest-package'};
# now commit all name.arch entries to the unneeded_na hash
$unneeded_na{$_} = 1 for @next_unneeded_na;
last if @blocked > $maxblocked;
}

return ('delayed', substr($delayed_errors, 2)) if $delayed_errors;
if ($markerdir && $myarch eq $buildarch) {
# update waiting_for markers
Expand Down
11 changes: 6 additions & 5 deletions src/backend/BSSched/EventHandler.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,13 +317,14 @@ sub event_scanprjbinaries {
my $prp = "$projid/$repoid";
my $arch = $ev->{'arch'} || $myarch;
delete $gctx->{'remotegbininfos'}->{"$prp/$arch"};
if ($ev->{'arch'}) {
# remote gbininfo retry event
my $changed_med = $gctx->{'changed_med'};
$changed_med->{$prp} = 2;
if ($remoteprojs->{$projid}) {
if ($ev->{'arch'}) {
# remote gbininfo retry event
my $changed_med = $gctx->{'changed_med'};
$changed_med->{$prp} = 2;
}
return;
}
return if $remoteprojs->{$projid};
if (defined($packid)) {
unlink("$prp/$myarch/$packid/.bininfo");
} else {
Expand Down
2 changes: 1 addition & 1 deletion src/backend/BSSched/RPC.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ sub xrpc_printstats {
print "running async RPC requests:\n";
for my $server (sort keys %$iswaiting_server) {
next unless %{$iswaiting_server->{$server} || {}};
print " - $server: ".scalar(keys %{$iswaiting_server->{$server}})." running, ".@{$iswaiting_serverload->{$server} || []}."/".@{$iswaiting_serverload_low->{$server} || []}." waiting\n";
print " - $server: ".scalar(keys %{$iswaiting_server->{$server}})." running, ".@{$iswaiting_serverload->{$server} || []}."+".@{$iswaiting_serverload_low->{$server} || []}." waiting\n";
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/backend/bs_repserver
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ sub getgbininfo {
my $prp = "$projid/$repoid";

die("getgbininfo: package filtering is not supported\n") if $cgi->{'package'};
my $gbininfo = BSRepServer::read_gbininfo("$reporoot/$prp/$arch", 1);
my $gbininfo = BSRepServer::read_gbininfo("$reporoot/$prp/$arch", 1, 1);
if ($cgi->{'withcode'}) {
my $ps = readpackstatus("$projid/$repoid/$arch");
my $code = ($ps || {})->{'packstatus'} || {};
Expand Down
Loading

0 comments on commit fa5742d

Please sign in to comment.