Skip to content

Commit

Permalink
[backend] support DoD container registries
Browse files Browse the repository at this point in the history 
Those DoD resources are different to repositories because there
is no way to request the complete state from the server.

Thus we collect the needed DoD resources in the scheduler and put
them into a "doddata.needed" file. The bs_dodup server then uses
this information to request data for each resource.
  • Loading branch information
@mlschroe
mlschroe committed May 27, 2021
1 parent 28479aa commit fbeaccf
Show file tree
Hide file tree
Showing 7 changed files with 477 additions and 13 deletions.
25 changes: 25 additions & 0 deletions src/backend/BSRepServer/DoD.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,4 +145,29 @@ sub fetchdodbinary {
return $localname;
}

sub setmissingdodresources {
my ($gdst, $id, $dodresources) = @_;
my $dir = "$gdst/:full";
die("no doddata\n") unless -s "$dir/doddata";
my @dr = sort(BSUtil::unify(@$dodresources));
my $needed = BSUtil::retrieve("$dir/doddata.needed", 1);
return if $needed && BSUtil::identical($needed->{$id} || [], \@dr);
my $fd;
if (!BSUtil::lockopen($fd, '>>', "$dir/doddata.needed", 1)) {
warn("$dir/doddata.needed: $!\n");
return;
}
$needed = {};
$needed = BSUtil::retrieve("$dir/doddata.needed", 1) || {} if -s "$dir/doddata.needed";
if (!@dr) {
delete $needed->{$id};
delete $needed->{''}->{$id} if $needed->{''};
} else {
$needed->{$id} = \@dr;
$needed->{''}->{$id} = time();
}
BSUtil::store("$dir/.doddata.needed", "$dir/doddata.needed", $needed);
close($fd);
}

1;
4 changes: 2 additions & 2 deletions src/backend/BSSched/BuildRepo.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -957,7 +957,7 @@ sub addrepo_scan {
my $doddata;
if ($BSConfig::enable_download_on_demand) {
$doddata = BSSched::DoD::get_doddata($gctx, $prp, $arch);
($dirty, $r) = BSSched::DoD::put_doddata_in_cache($pool, $prp, $r, $doddata, $dir);
($dirty, $r) = BSSched::DoD::put_doddata_in_cache($gctx, $doddata, $pool, $prp, $r, $dir);
}

my @bins;
Expand Down Expand Up @@ -994,7 +994,7 @@ sub addrepo_scan {
return undef unless $r;
# write solv file (unless alien arch)
if ($dirty && $arch eq $gctx->{'arch'}) {
@bins = BSSched::DoD::clean_obsolete_dodpackages($pool, $r, $dir, @bins) if $doddata;
@bins = BSSched::DoD::clean_obsolete_dodpackages($gctx, $doddata, $pool, $prp, $r, $dir, @bins) if $doddata;
writesolv("$dir.solv.new", "$dir.solv", $r);
}
$repocache->setcache($prp, $arch) if $repocache;
Expand Down
72 changes: 64 additions & 8 deletions src/backend/BSSched/Checker.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ use BSSched::PublishRepo;
use BSSched::BuildJob;
use BSSched::Access;
use BSSched::Remote; # for addrepo_remote
use BSSched::DoD; # for signalmissing
use BSSched::EventSource::Directory;

use BSSched::BuildJob::Aggregate;
Expand Down Expand Up @@ -241,6 +242,23 @@ sub check_remote_repo_error {
return undef;
}

sub neededdodresources {
my ($ctx, $repotype) = @_;
return () unless ($repotype || '') eq 'registry';
my $projpacks = $ctx->{'gctx'}->{'projpacks'};
my $projid = $ctx->{'project'};
my $repoid = $ctx->{'repository'};
my $proj = $projpacks->{$projid} || {};
my $pdatas = $proj->{'package'} || {};
my %needed;
for my $pdata (values %$pdatas) {
my $info = (grep {$_->{'repository'} eq $repoid} @{$pdata->{'info'} || []})[0];
next unless $info;
$needed{$_} = 1 for grep {/^container:/} @{$info->{'dep'} || []};
}
return sort keys %needed;
}

sub setup {
my ($ctx) = @_;
my $prp = $ctx->{'prp'};
Expand Down Expand Up @@ -365,8 +383,8 @@ sub setup {
return (0, 'crosshostarch mismatch');
}

# setup host data if doing cross builds
if ($crosshostarch && $crosshostarch ne $myarch) {
# doing cross builds, setup host data
if (!grep {$_ eq $crosshostarch} @{$repo->{'arch'} || []}) {
return ('broken', "host arch $crosshostarch missing in repo architectures");
}
Expand Down Expand Up @@ -1137,6 +1155,16 @@ sub checkpkgs {
next;
}

# check if we have all the dod resources
if ($ctx->{'missingdodresources'}) {
my @missing = grep {$ctx->{'missingdodresources'}->{$_}} @{$info->{'dep'} || []};
if (@missing) {
$packstatus{$packid} = 'blocked';
$packerror{$packid} = "waiting for dod resources to appear: @missing";
next;
}
}

# all checks ok, dispatch to handler
my $handler = $handlers{$buildtype} || $handlers{default};
my ($astatus, $aerror) = $handler->check($ctx, $packid, $pdata, $info, $buildtype);
Expand Down Expand Up @@ -1381,31 +1409,59 @@ sub checkprpaccess {
return BSSched::Access::checkprpaccess($ctx->{'gctx'}, $prp, $ctx->{'prp'});
}

sub checkdodresources {
my ($ctx, $prp, $arch, $r) = @_;
return unless defined &BSSolv::repo::dodresources;
my $gctx = $ctx->{'gctx'};
my $dodrepotype = BSSched::ProjPacks::getdodrepotype($gctx, $prp);
return unless $dodrepotype;
my $dodresources = $ctx->{'dodresources'}->{$dodrepotype};
if (!$dodresources) {
$dodresources = $ctx->{'dodresources'}->{$dodrepotype} = [ neededdodresources($ctx, $dodrepotype) ];
}
return unless @{$dodresources || []};
return if $arch ne $gctx->{'arch'}; # not yet
my %reporesources = map {$_ => 1} $r->dodresources();
my @missing = grep {!$reporesources{$_}} @$dodresources;
return unless @missing;
print " missig dod resources: @missing\n";
$ctx->{'missingdodresources'} = { %{$ctx->{'missingdodresources'} || {}}, map {$_ => 1} @missing };
my $alldodresources = BSSched::ProjPacks::neededdodresources($gctx, $prp) || [];
$alldodresources = [ BSUtil::unify(@$alldodresources, @$dodresources) ];
BSSched::DoD::signalmissing($ctx, $prp, $arch, $alldodresources);
}

sub addrepo {
my ($ctx, $pool, $prp, $arch) = @_;
my $gctx = $ctx->{'gctx'};
$arch ||= $gctx->{'arch'};

# first check the cache
my $r = $gctx->{'repodatas'}->addrepo($pool, $prp, $arch);
return $r if $r || !defined($r);
return undef unless defined $r;
# make sure that we know all of the resources we need
checkdodresources($ctx, $prp, $arch, $r) if $r && $r->dodurl();
return $r if $r;

# not in cache. scan/fetch.
my ($projid, $repoid) = split('/', $prp, 2);
my $remoteprojs = $gctx->{'remoteprojs'};
if ($remoteprojs->{$projid}) {
return BSSched::Remote::addrepo_remote($ctx, $pool, $prp, $arch, $remoteprojs->{$projid});
}
if ($arch ne $gctx->{'arch'}) {
$r = BSSched::Remote::addrepo_remote($ctx, $pool, $prp, $arch, $remoteprojs->{$projid});
} elsif ($arch ne $gctx->{'arch'}) {
my $alien_cache = $ctx->{'alien_repo_cache'};
$alien_cache = $ctx->{'alien_repo_cache'} = {} unless $alien_cache;
$r = $pool->repofromstr($prp, $alien_cache->{"$prp/$arch"}) if exists $alien_cache->{"$prp/$arch"};
if (!$r) {
# needs some mem, but it's hopefully worth it
$r = BSSched::BuildRepo::addrepo_scan($gctx, $pool, $prp, $arch);
# needs some mem, but it's hopefully worth it
$alien_cache->{"$prp/$arch"} = $r->tostr() if $r;
}
return $r;
} else {
$r = BSSched::BuildRepo::addrepo_scan($gctx, $pool, $prp, $arch);
}
return BSSched::BuildRepo::addrepo_scan($gctx, $pool, $prp, $arch);
checkdodresources($ctx, $prp, $arch, $r) if $r && $r->dodurl();
return $r;
}

sub read_gbininfo {
Expand Down
100 changes: 98 additions & 2 deletions src/backend/BSSched/DoD.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ use Digest::MD5 ();

use BSUtil;
use BSSched::RPC;
use BSSched::Blobstore;
use BSXML;
use BSHTTP;

Expand Down Expand Up @@ -82,7 +83,7 @@ sub readparsed {
=cut

sub put_doddata_in_cache {
my ($pool, $prp, $cache, $doddata, $dir) = @_;
my ($gctx, $doddata, $pool, $prp, $cache, $dir) = @_;
if (!$doddata) {
return (0, $cache) if !$cache || !$cache->dodurl();
$cache->updatedoddata() if defined &BSSolv::repo::updatedoddata;
Expand Down Expand Up @@ -119,7 +120,7 @@ sub put_doddata_in_cache {
=cut

sub clean_obsolete_dodpackages {
my ($pool, $cache, $dir, @bins) = @_;
my ($gctx, $doddata, $pool, $prp, $cache, $dir, @bins) = @_;

return @bins unless defined &BSSolv::repo::pkgpaths;
my %paths = $cache->pkgpaths();
Expand Down Expand Up @@ -166,6 +167,16 @@ sub clean_obsolete_dodpackages {
unlink("$dir/$path");
}
$cache->updatefrombins($dir, @nbins) if $nbinsdirty;

if ($clean_blobs && defined(&BSSolv::repo::getdodblobs)) {
my %blobs = map {$_ => 1} $cache->getdodblobs();
for my $blob (sort(grep {/^_blob\.(.*)$/ && !$blobs{$1}} ls($dir))) {
print " - :full/$blob [DoD cleanup]\n";
unlink("$dir/$blob");
BSSched::Blobstore::blobstore_chk($gctx, $blob);
}
}

return @nbins;
}

Expand All @@ -177,6 +188,7 @@ sub clean_obsolete_dodpackages {

sub dodcheck {
my ($ctx, $pool, $arch, @pkgs) = @_;
return unless @pkgs;
$ctx = $ctx->{'realctx'} if $ctx->{'realctx'}; # we need the real one to add entries
my %names;
if (defined &BSSolv::repo::dodcookie) {
Expand Down Expand Up @@ -384,4 +396,88 @@ sub init_doddata {
BSUtil::touch("$dodsdir/.changed") if $changed && -d $dodsdir;
}


=head2 signalmissing_resume- TODO: add summary
TODO: add description
=cut

sub signalmissing_resume {
my ($ctx, $handle, $error) = @_;
my ($projid, $repoid) = split('/', $handle->{'_prp'}, 2);
my $gctx = $ctx->{'gctx'};
if ($error) {
if (BSSched::RPC::is_transient_error($error)) {
$gctx->{'retryevents'}->addretryevent({'type' => 'repository', 'project' => $projid, 'repository' => $repoid, 'arch' => $gctx->{'arch'}});
}
return;
}
}

=head2 signalmissing - inform the dodup tool about missing dod resources
=cut

sub signalmissing {
my ($ctx, $prp, $arch, $dodresources) = @_;
return unless @{$dodresources || []};
my $gctx = $ctx->{'gctx'};
my $projpacks = $gctx->{'projpacks'};
my ($projid, $repoid) = split('/', $prp, 2);

if (!$projpacks->{$projid}) {
# dod from different partition, use repo server
my $remoteprojs = $gctx->{'remoteprojs'};
my $proj = $remoteprojs->{$projid};
return unless $proj;
return unless $proj->{'partition'}; # not supported yet
my $server = $proj->{'partition'} ? $proj->{'remoteurl'} : $BSConfig::srcserver;
my $param = {
'uri' => "$server/build/$prp/$arch",
'request' => 'POST',
'receiver' => \&BSHTTP::null_receiver,
'async' => {
'_resume' => \&signalmissing_resume,
'_changetype' => 'low',
'_prp' => $prp,
},
};
my @args = 'view=missingdodresources';
push @args, map {"resource=$_"} @$dodresources;
push @args, "partition=$BSConfig::partition" if $BSConfig::partition;
eval { $ctx->xrpc("missingdodresources/$prp", $param, undef, @args) };
if ($@) {
warn($@);
$gctx->{'retryevents'}->addretryevent({'type' => 'repository', 'project' => $projid, 'repository' => $repoid, 'arch' => $arch });
}
return;
}

# local dod, directly modify
my $id = "$gctx->{'arch'}";
$id = "$BSConfig::partition/$id" if $BSConfig::partition;
my $dir = "$gctx->{'reporoot'}/$prp/$arch/:full";
mkdir_p($dir);
my @dr = sort(BSUtil::unify(@$dodresources));
my $needed = BSUtil::retrieve("$dir/doddata.needed", 1);
return $needed if $needed && BSUtil::identical($needed->{$id} || [], \@dr);
my $fd;
if (!BSUtil::lockopen($fd, '>>', "$dir/doddata.needed", 1)) {
warn("$dir/doddata.needed: $!\n");
return;
}
$needed = {};
$needed = BSUtil::retrieve("$dir/doddata.needed", 1) || {} if -s "$dir/doddata.needed";
if (!@dr) {
delete $needed->{$id};
delete $needed->{''}->{$id} if $needed->{''};
} else {
$needed->{$id} = \@dr;
$needed->{''}->{$id} = time();
}
BSUtil::store("$dir/.doddata.needed", "$dir/doddata.needed", $needed);
close($fd);
}

1;
45 changes: 45 additions & 0 deletions src/backend/BSSched/ProjPacks.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1632,6 +1632,51 @@ sub getconfig {
return $c;
}

=head2 getdodrepotype - get the repotype of a dod prp
=cut

sub getdodrepotype {
my ($gctx, $prp) = @_;
my $projpacks = $gctx->{'projpacks'};
my $remoteprojs = $gctx->{'remoteprojs'};
my $myarch = $gctx->{'arch'};

my ($projid, $repoid) = split('/', $prp, 2);
my $proj = $projpacks->{$projid};
$proj = $remoteprojs->{$projid} if !$proj || $proj->{'remoteurl'};
return undef unless $proj;
my $repo = (grep {$_->{'name'} eq $repoid} @{$proj->{'repository'} || []})[0];
return undef unless $repo && $repo->{'download'};
my $doddata = (grep {($_->{'arch'} || '') eq $myarch} @{$repo->{'download'} || []})[0];
return $doddata ? $doddata->{'repotype'} || 'unset' : undef;
}

=head2 neededdodresources - get all requested dod resources for a dod repo
=cut

sub neededdodresources {
my ($gctx, $prp) = @_;
my $dodrepotype = getdodrepotype($gctx, $prp);
return [] unless $dodrepotype eq 'registry';
my $projpacks = $gctx->{'projpacks'};
my $rprpdeps = $gctx->{'rprpdeps'}->{$prp};
return [] unless $rprpdeps;
my %needed;
for my $aprp (@$rprpdeps) {
my ($aprojid, $arepoid) = split('/', $aprp, 2);
my $aproj = $projpacks->{$aprojid};
next unless $aproj;
my $pdatas = $aproj->{'package'} || {};
for my $pdata (values %$pdatas) {
my $info = (grep {$_->{'repository'} eq $arepoid} @{$pdata->{'info'} || []})[0];
next unless $info;
$needed{$_} = 1 for grep {/^container:/} @{$info->{'dep'} || []};
}
}
return [ sort keys %needed ];
}

=head2 orderpackids - sort package containers
Expand Down

0 comments on commit fbeaccf

Please sign in to comment.