From b73f00b5e1dcb6b78e6f468eed6906d4eb50368a Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 15 Sep 2023 10:38:18 +0200 Subject: [PATCH] [backend] BSSched: new cyclic dependency handling Add a new cycsort function that orders the packages from a cycle. Change handlecycle to just do two phases: phase 1 handles source changes and cyclevel calculation, phase 2 is for meta changes. --- src/backend/BSSched/BuildJob/Package.pm | 35 ++++++++---- src/backend/BSSched/Checker.pm | 76 +++++++++++++------------ 2 files changed, 65 insertions(+), 46 deletions(-) diff --git a/src/backend/BSSched/BuildJob/Package.pm b/src/backend/BSSched/BuildJob/Package.pm index eb92ac258df..28212237329 100644 --- a/src/backend/BSSched/BuildJob/Package.pm +++ b/src/backend/BSSched/BuildJob/Package.pm @@ -102,7 +102,7 @@ sub check { my @blocked = grep {$notready->{$dep2src->{$_}}} @$edeps; @blocked = () if $repo->{'block'} && $repo->{'block'} eq 'never'; # check if cycle builds are in progress - if ($incycle && $incycle == 3) { + if ($incycle == 3) { push @blocked, 'cycle' unless @blocked; if ($ctx->{'verbose'}) { print " - $packid ($buildtype)\n"; @@ -112,10 +112,14 @@ sub check { return ('blocked', join(', ', @blocked)); } # prune cycle packages from blocked - if ($incycle) { + if ($incycle > 1) { + my $cyclevel = $ctx->{'cyclevel'}; my $pkg2src = $ctx->{'pkg2src'} || {}; - my %cycs = map {($pkg2src->{$_} || $_) => 1} @{$ctx->{'cychash'}->{$packid}}; - @blocked = grep {!$cycs{$dep2src->{$_}}} @blocked; + my $level = $cyclevel->{$packid}; + if ($level) { + my %cycs = map {($pkg2src->{$_} || $_) => ($cyclevel->{$_} || 1)} @{$ctx->{'cychash'}->{$packid}}; + @blocked = grep {($cycs{$dep2src->{$_}} || 0) < $level} @blocked; + } } if (@blocked) { # print " - $packid ($buildtype)\n"; @@ -202,11 +206,6 @@ sub check { goto relsynccheck; } # more work, check if dep rpm changed - if ($incycle == 1) { - # print " - $packid ($buildtype)\n"; - # print " in cycle, no source change...\n"; - return ('done'); - } my $check = substr($mylastcheck, 32, 32); # metamd5 my $pool = $ctx->{'pool'}; my $pool_host = $ctx->{'pool_host'}; @@ -297,6 +296,21 @@ sub check { close F; chomp @meta; } + if ($incycle == 1) { + # calculate cyclevel + my $level; + if (defined &BSSolv::diffdepth_meta) { + $level = BSSolv::diffdepth_meta(\@new_meta, \@meta); + } else { + $level = BSBuild::diffdepth(\@new_meta, \@meta); + } + $ctx->{'cyclevel'}->{$packid} = $level; + if ($level > 1) { + # print " - $packid ($buildtype)\n"; + # print " in cycle, no source change...\n"; + return ('done'); # postpone till phase 2 + } + } if ($rebuildmethod eq 'direct') { @meta = grep {!/\//} @meta; @new_meta = grep {!/\//} @new_meta; @@ -315,10 +329,11 @@ sub check { } my @diff = BSSched::BuildJob::diffsortedmd5(\@meta, \@new_meta); my $reason = BSSched::BuildJob::sortedmd5toreason(@diff); + my $levelstr = $ctx->{'cyclevel'}->{$packid} ? " (cyclevel $ctx->{'cyclevel'}->{$packid})" : ''; if ($ctx->{'verbose'}) { print " - $packid ($buildtype)\n"; print " $_\n" for @diff; - print " meta change, start build\n"; + print " meta change, start build$levelstr\n"; } return ('scheduled', [ { 'explain' => 'meta change', 'packagechange' => $reason }, $hdeps ] ); } diff --git a/src/backend/BSSched/Checker.pm b/src/backend/BSSched/Checker.pm index fb8bae2b2a4..eba7f47df39 100644 --- a/src/backend/BSSched/Checker.pm +++ b/src/backend/BSSched/Checker.pm @@ -934,47 +934,50 @@ sub prune_packstatus_finished { sub handlecycle { my ($ctx, $packid, $cpacks, $cycpass) = @_; - my $incycle = 0; my $cychash = $ctx->{'cychash'}; return ($packid, 0) unless $cychash->{$packid}; - $incycle = $cycpass->{$packid} || 0; - - if (!$incycle) { - # starting pass 1 (incycle == 1) - my @cycp = @{$cychash->{$packid}}; - unshift @$cpacks, $cycp[0]; # pass3 - unshift @$cpacks, @cycp; # pass2 - unshift @$cpacks, @cycp; # pass1 + my $incycle = $cycpass->{$packid} || 0; + return ($packid, $incycle) if $incycle > 0; # still in pass + my @cycp = @{$cychash->{$packid}}; + $incycle = -$incycle + 1; # start next pass + $cycpass->{$_} = $incycle for @cycp; + if ($incycle == 1) { + unshift @$cpacks, $cycp[0]; + unshift @$cpacks, @cycp; + $packid = shift @$cpacks; + $cycpass->{$packid} = -1; # set pass1 endmarker + } elsif ($incycle == 2) { + my $cyclevel = $ctx->{'cyclevel'}; + unshift @$cpacks, sort {($cyclevel->{$a} || 0) <=> ($cyclevel->{$b} || 0)} @cycp; + $packid = shift @$cpacks; + $cycpass->{$packid} = -2; # set pass2 endmarker + } elsif ($incycle == 3) { + unshift @$cpacks, @cycp; $packid = shift @$cpacks; - $incycle = 1; - $cycpass->{$_} = $incycle for @cycp; - $cycpass->{$packid} = -1; # pass1 ended - } elsif ($incycle == -1) { - # starting pass 2 (incycle will be 2 or 3) - my @cycp = @{$cychash->{$packid}}; - my $building = $ctx->{'building'}; - $incycle = (grep {$building->{$_}} @cycp) ? 3 : 2; - $cycpass->{$_} = $incycle for @cycp; - $cycpass->{$packid} = -2; # pass2 ended - } elsif ($incycle == -2) { - # starting pass 3 (incycle == 4) - my @cycp = @{$cychash->{$packid}}; - $incycle = 4; - $cycpass->{$_} = $incycle for @cycp; - # propagate notready/unfinished to all cycle packages - my $notready = $ctx->{'notready'}; - my $pkg2src = $ctx->{'pkg2src'} || {}; - if (grep {$notready->{$pkg2src->{$_} || $_}} @cycp) { - $notready->{$pkg2src->{$_} || $_} ||= 1 for @cycp; - } - my $unfinished = $ctx->{'unfinished'}; - if (grep {$unfinished->{$pkg2src->{$_} || $_}} @cycp) { - $unfinished->{$pkg2src->{$_} || $_} ||= 1 for @cycp; - } } return ($packid, $incycle); } +sub cycsort { + my ($pkg2dep, $dep2src, $pkg2src, @cyc) = @_; + @cyc = BSUtil::unify(sort(@cyc)); + my %d; + my %cdeps; + for my $pkg (@cyc) { + $d{$dep2src->{$_} || $_}->{$pkg} = 1 for @{$pkg2dep->{$pkg}}; + } + # remove all bi-directional edges + my %ign; + for my $pkg (@cyc) { + $ign{$pkg}->{$_} = 1 for keys %{$d{$pkg2src->{$pkg}} || {}}; + } + for my $pkg (@cyc) { + $_ ne $pkg && !$ign{$_}->{$pkg} and push @{$cdeps{$_}}, $pkg for keys %{$d{$pkg2src->{$pkg}} || {}}; + } + @cyc = BSSolv::depsort(\%cdeps, undef, undef, @cyc); + return @cyc; +} + sub checkpkgs { my ($ctx) = @_; @@ -1015,12 +1018,13 @@ sub checkpkgs { $ctx->{'nharder'} = 0; $ctx->{'building'} = \%building; $ctx->{'unfinished'} = \%unfinished; + $ctx->{'cyclevel'} = {}; # now build cychash mapping packages to all other cycle members for my $cyc (@{$ctx->{'sccs'} || $ctx->{'cycles'} || []}) { next if @$cyc < 2; # just in case my @c = map {@{$cychash{$_} || [ $_ ]}} @$cyc; - @c = BSUtil::unify(sort(@c)); + @c = cycsort($ctx->{'edeps'}, $ctx->{'dep2src'}, $ctx->{'pkg2src'}, @c); $cychash{$_} = \@c for @c; } @@ -1053,7 +1057,6 @@ sub checkpkgs { my $incycle = 0; if ($cychash{$packid}) { ($packid, $incycle) = handlecycle($ctx, $packid, \@cpacks, \%cycpass); - next if $incycle == 4; # ignore after pass1/2 next if $packstatus{$packid} && $packstatus{$packid} ne 'done' && $packstatus{$packid} ne 'succeeded' && $packstatus{$packid} ne 'failed'; # already decided } $ctx->{'incycle'} = $incycle; @@ -1204,6 +1207,7 @@ sub checkpkgs { $notready->{$pname} = 1 if $useforbuildenabled; $unfinished{$pname} = 1; $packstatus{$packid} = 'scheduled'; + # we may also want to set the cyclevel next; }