Skip to content

Commit

Permalink
[backend] make Docker building work for partitions
Browse files Browse the repository at this point in the history 
This is not an easy task. The problem is that for Docker builds,
the container provides extra path elements. Thus we do not know
the complete path when getting the project data (getprojpack).
This results in missing entries in our remotemap.

This commit tries to deal with that problem by requesting remotemap
updates from the build job generation code. We also need a new
remotemissing hash so that we do not request non-existing projects
over and over again.

For now we change the info element from the package data by
appending the new path elements. This has the advantage that
the project sorting function will see the new dependencies and
do the right thing. Patching the info data is a bit nasty, though.
  • Loading branch information
@mlschroe
mlschroe committed Jun 27, 2017
1 parent 6b3559f commit 59e2829
Show file tree
Hide file tree
Showing 6 changed files with 138 additions and 20 deletions.
13 changes: 10 additions & 3 deletions src/backend/BSRepServer/Checker.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,16 @@ sub getconfig {
return BSRepServer::ProjPacks::getconfig($ctx->{'gctx'}, $projid, $repoid, $arch, $configpath);
}

sub get_path_projpacks {
my ($ctx, $projid, $path) = @_;
return BSRepServer::ProjPacks::get_path_projpacks($ctx->{'gctx'}, $projid, $path);
sub append_info_path {
my ($ctx, $info, $path) = @_;
# make sure we know about the path elements
BSRepServer::ProjPacks::get_path_projpacks($ctx->{'gctx'}, $ctx->{'project'}, $path);
# append path to info
splice(@{$info->{'path'}}, -$info->{'extrapathlevel'}) if $info->{'extrapathlevel'};
delete $info->{'extrapathlevel'};
push(@{$info->{'path'}}, @$path);
$info->{'extrapathlevel'} = @$path if @$path;
return 1;
}

sub setup {
Expand Down
23 changes: 9 additions & 14 deletions src/backend/BSSched/BuildJob/Docker.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ sub urlmapper {
my ($ctx, $url) = @_;
$url =~ s/\/+$//;
my $kiwiurlmapcache = {};
for my $prp (%{$BSConfig::prp_ext_map || {}}) {
for my $prp (sort keys %{$BSConfig::prp_ext_map || {}}) {
my $u = $BSConfig::prp_ext_map->{$prp};
$u =~ s/\/+$//;
$kiwiurlmapcache->{$u} = $prp;
Expand Down Expand Up @@ -113,7 +113,6 @@ sub check {

my @deps = @{$info->{'dep'} || []};

my @newpath;
my $cdep; # container dependency
my $cprp; # container prp
my $cbdep; # container bdep for job
Expand Down Expand Up @@ -143,11 +142,10 @@ sub check {

# generate bdep entry
$cbdep = {'name' => $cdeps[0], 'noinstall' => 1};
($cbdep->{'project'}, $cbdep->{'repository'}) = split('/', $cprp, 2);
$cprp = $cpool->pkg2reponame($p);
$cmeta = $cpool->pkg2pkgid($p) . " $cprp/$cdeps[0]";
if ($ctx->{'dobuildinfo'}) {
($cbdep->{'project'}, $cbdep->{'repository'}) = split('/', $cprp, 2) if $cprp;
($cbdep->{'project'}, $cbdep->{'repository'}) = split('/', $cprp, 2);
my $d = $cpool->pkg2data($p);
$cbdep->{'epoch'} = $d->{'epoch'} if $d->{'epoch'};
$cbdep->{'version'} = $d->{'version'};
Expand All @@ -156,7 +154,8 @@ sub check {
$cbdep->{'hdrmd5'} = $d->{'hdrmd5'} if $d->{'hdrmd5'};
}

# add container repositories
# append container repositories to path
my @newpath;
if (defined &BSSolv::pool::pkg2annotation) {
my $annotation_xml = $cpool->pkg2annotation($p);
if ($annotation_xml) {
Expand All @@ -183,11 +182,11 @@ sub check {
}
}
}
$ctx->get_path_projpacks($projid, \@newpath) if @newpath;
my $r = $ctx->append_info_path($info, \@newpath);
return ('delayed', 'remotemap entry missing') unless $r;
}
unshift @newpath, @{$info->{'path'} || []};

my @aprps = map {"$_->{'project'}/$_->{'repository'}"} @newpath;
my @aprps = map {"$_->{'project'}/$_->{'repository'}"} @{$info->{'path'} || []};

# get config from docker path
my @configpath = @aprps;
Expand Down Expand Up @@ -294,7 +293,7 @@ sub check {
push @new_meta, $cmeta if $cmeta;
@new_meta = sort {substr($a, 34) cmp substr($b, 34)} @new_meta;
unshift @new_meta, map {"$_->{'srcmd5'} $_->{'project'}/$_->{'package'}"} @{$info->{'extrasource'} || []};
my ($state, $data) = BSSched::BuildJob::metacheck($ctx, $packid, $pdata, 'docker', \@new_meta, [ $bconf, \@edeps, $pool, \%dep2pkg, $cbdep, $cprp, \@newpath ]);
my ($state, $data) = BSSched::BuildJob::metacheck($ctx, $packid, $pdata, 'docker', \@new_meta, [ $bconf, \@edeps, $pool, \%dep2pkg, $cbdep, $cprp]);
if ($BSConfig::enable_download_on_demand && $state eq 'scheduled') {
my $dods = BSSched::DoD::dodcheck($ctx, $pool, $myarch, @edeps);
return ('blocked', $dods) if $dods;
Expand All @@ -317,17 +316,13 @@ sub build {
my $edep2pkg = $data->[3];
my $cbdep = $data->[4];
my $cprp = $data->[5];
my $newpath = $data->[6];
my $reason = $data->[7];
my $reason = $data->[6];

my $gctx = $ctx->{'gctx'};
my $projid = $ctx->{'project'};
my $repoid = $ctx->{'repository'};
my $repo = $ctx->{'repo'};

# fixup path in info
$info = { %$info, 'path' => $newpath } if @$newpath;

if (!@{$repo->{'path'} || []}) {
# repo has no path, use docker repositories also for docker system setup
my $xp = BSSolv::expander->new($epool, $bconf);
Expand Down
54 changes: 52 additions & 2 deletions src/backend/BSSched/Checker.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1064,8 +1064,58 @@ sub getconfig {
return BSSched::ProjPacks::getconfig($ctx->{'gctx'}, $projid, $repoid, $arch, $configpath);
}

sub get_path_projpacks {
my ($ctx, $projid, $path) = @_;
sub append_info_path {
my ($ctx, $info, $path) = @_;

my $gctx = $ctx->{'gctx'};
my $projid = $ctx->{'project'};

# append path to info
my @oldpath;
if ($info->{'extrapathlevel'}) {
@oldpath = @{$info->{'path'}}; # create copy
@oldpath = splice(@oldpath, -$info->{'extrapathlevel'});
}
if (!BSUtil::identical(\@oldpath, $path)) {
print "append_info_path: different path\n";
# path has changed. remove old one
splice(@{$info->{'path'}}, -$info->{'extrapathlevel'}) if $info->{'extrapathlevel'};
delete $info->{'extrapathlevel'};
# add new one
push @{$info->{'path'}}, @$path;
$info->{'extrapathlevel'} = @$path if @$path;
$gctx->{'prj_depsort_needed'} = 1;
} else {
print "append_info_path: same path\n";
}

# check if we have missing remotemap entries
my $projpacks = $gctx->{'projpacks'};
my $remoteprojs = $gctx->{'remoteprojs'};
my $remotemissing = $gctx->{'remotemissing'};
my $ret = 1;
my @missing;
for my $pe (@$path) {
my $pr = $pe->{'project'};
next if $projpacks->{$pr} || ($remoteprojs->{$pr} && defined($remoteprojs->{$pr}->{'config'})) || $remotemissing->{$pr};
$ret = 0;
next if defined $remotemissing->{$pr};
push @missing, $pr;
}
@missing = BSUtil::unify(@missing);
for my $projid (@missing) {
my $asyncmode = $gctx->{'asyncmode'};
my $async;
if ($asyncmode) {
$async = {
'_changeprp' => $ctx->{'changeprp'},
'_changetype' => $ctx->{'changetype'} || 'high',
'_changelevel' => $ctx->{'changelevel'} || 1,
};
}
BSSched::ProjPacks::get_remoteproject($gctx, $async, $projid);
}
return $ret;
}

1;
46 changes: 46 additions & 0 deletions src/backend/BSSched/ProjPacks.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1340,4 +1340,50 @@ sub runningfetchprojpacks {
return \%running;
}

sub get_remoteproject_resume {
my ($gctx, $handle, $error, $projpacksin) = @_;
my $projid = $handle->{'_projid'};
delete $gctx->{'remotemissing'}->{$projid};
if ($error) {
chomp $error;
warn("$error\n");
return;
}
BSSched::Remote::remotemap2remoteprojs($gctx, $projpacksin->{'remotemap'});
$gctx->{'remotemissing'}->{$projid} = 1 if !$gctx->{'projpacks'}->{$projid} && !$gctx->{'remoteprojs'}->{$projid};
BSSched::Lookat::setchanged($gctx, $handle->{'_changeprp'}, $handle->{'_changetype'}, $handle->{'_changelevel'});
}

sub get_remoteproject {
my ($gctx, $doasync, $projid) = @_;

my $myarch = $gctx->{'arch'};
print "getting data for remote project '$projid' from $BSConfig::srcserver\n";
my @args;
push @args, "partition=$BSConfig::partition" if $BSConfig::partition;
push @args, "project=$projid";
my $param = {
'uri' => "$BSConfig::srcserver/getprojpack",
};
if ($doasync) {
$param->{'async'} = { %$doasync, '_resume' => \&get_remoteproject_resume, '_projid' => $projid};
}
my $projpacksin;
eval {
if ($usestorableforprojpack) {
$projpacksin = $gctx->{'rctx'}->xrpc($gctx, $projid, $param, \&BSUtil::fromstorable, 'view=storable', 'withconfig', 'withremotemap', 'remotemaponly', "arch=$myarch", @args);
} else {
$projpacksin = $gctx->{'rctx'}->xrpc($gctx, $projid, $param, $BSXML::projpack, 'withconfig', 'withremotemap', 'remotemaponly', "arch=$myarch", @args);
}
};
return 0 if !$@ && $projpacksin && $param->{'async'};
delete $gctx->{'remotemissing'}->{$projid};
if ($@) {
warn($@) if $@;
return;
}
BSSched::Remote::remotemap2remoteprojs($gctx, $projpacksin->{'remotemap'});
$gctx->{'remotemissing'}->{$projid} = 1 if !$gctx->{'projpacks'}->{$projid} && !$gctx->{'remoteprojs'}->{$projid};
}

1;
17 changes: 16 additions & 1 deletion src/backend/BSSched/Remote.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ sub updateremoteprojs {
my ($gctx, $needremoteproj) = @_;

my $remoteprojs = $gctx->{'remoteprojs'};
my $remotemissing = $gctx->{'remotemissing'};
for my $projid (keys %$remoteprojs) {
my $or = $remoteprojs->{$projid};
next if $or && $or->{'partition'}; # XXX how do we update them?
Expand All @@ -154,7 +155,7 @@ sub updateremoteprojs {
}
for my $projid (sort keys %$needremoteproj) {
my $r = $needremoteproj->{$projid};
fetchremoteproj($gctx, $r, $projid) if $r && !$remoteprojs->{$projid};
fetchremoteproj($gctx, $r, $projid) if $r && !$remoteprojs->{$projid} && !exists($remotemissing->{$projid});
}
}

Expand Down Expand Up @@ -260,6 +261,8 @@ sub fetchremoteproj {
return undef unless $proj && $proj->{'remoteurl'} && $proj->{'remoteproject'};
my $remoteprojs = $gctx->{'remoteprojs'};
return $remoteprojs->{$projid} if exists $remoteprojs->{$projid};
my $remotemissing = $gctx->{'remotemissing'};
return undef if $remotemissing->{$projid};
return fetchremote_sync($gctx, $projid); # force in missing entry
}

Expand Down Expand Up @@ -314,6 +317,18 @@ sub remotemap2remoteprojs {
}
$remoteprojs->{$projid} = $proj;
}
# update remotemissing map
my $projpacks = $gctx->{'projpacks'};
for my $projid (keys %{$gctx->{'remotemissing'}}) {
if ($projpacks->{$projid}) {
delete $gctx->{'remotemissing'}->{$projid};
} elsif ($remoteprojs->{$projid}) {
if (!defined($remoteprojs->{$projid}->{'config'})) {
next unless $gctx->{'remotemissing'}->{$projid}; # keep the "in progress" flag
}
delete $gctx->{'remotemissing'}->{$projid};
}
}
}

###########################################################################
Expand Down
5 changes: 5 additions & 0 deletions src/backend/bs_sched
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,7 @@ my $gctx = {
'projpacks' => undef, # data of all local projects
'channeldata' => {}, # global channel data unificator to save memory
'remoteprojs' => {}, # remote project cache
'remotemissing' => {}, # missing remote projects cache
'projsuspended' => {}, # project is suspended for now

# lastcheck cache
Expand Down Expand Up @@ -1111,6 +1112,10 @@ eval {
writeschedulerinfo($gctx);
$lastschedinfo = $now;
}

if (delete $gctx->{'prj_depsort_needed'}) {
BSSched::ProjPacks::get_projpacks_postprocess($gctx);
}
}

};
Expand Down

0 comments on commit 59e2829

Please sign in to comment.