Skip to content

Commit

Permalink
[backend] support download on demand of containers
Browse files Browse the repository at this point in the history
  • Loading branch information
@mlschroe
mlschroe committed May 17, 2021
1 parent fe57ef9 commit e185c51
Show file tree
Hide file tree
Showing 3 changed files with 167 additions and 10 deletions.
126 changes: 125 additions & 1 deletion src/backend/BSRepServer/DoD.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,17 @@
#
package BSRepServer::DoD;

use Digest::SHA ();

use BSWatcher ':https';
use BSVerify;
use BSHandoff;
use BSContar;
use BSStdServer;
use BSUtil;

use BSRepServer::Containertar;
use BSRepServer::Containerinfo;

use Build;

Expand Down Expand Up @@ -51,14 +58,131 @@ sub is_wanted_dodbinary {
return 1;
}

sub is_wanted_dodcontainer {
my ($pool, $p, $path, $doverify) = @_;
my $q = BSUtil::retrieve("$path.obsbinlnk", 1);
return 0 unless $q;
my $data = $pool->pkg2data($p);
return 0 if $data->{'name'} ne $q->{'name'} || $data->{'version'} ne $q->{'version'};
BSVerify::verify_nevraquery($q) if $doverify; # just in case
return 1;
}

sub blob_matches_digest {
my ($tmp, $digest) = @_;
my $ctx;
$ctx = Digest::SHA->new($1) if $digest =~ /^sha(256|512):/;
return 0 unless $ctx;
my $fd;
return 0 unless open ($fd, '<', $tmp);
$ctx->addfile($fd);
close($fd);
return (split(':', $digest, 2))[1] eq $ctx->hexdigest() ? 1 : 0;
}

sub fetchdodcontainer {
my ($gdst, $pool, $repo, $p, $handoff) = @_;
my $container = $pool->pkg2name($p);
$container =~ s/^container://;

my $pkgname = $container;
$pkgname =~ s/\//_/g;
$pkgname = "_$pkgname" if $pkgname =~ /^_/;
BSVerify::verify_filename($pkgname);
BSVerify::verify_simple($pkgname);
my $dir = "$gdst/:full";

if (-e "$dir/$pkgname.obsbinlnk" && -e "$dir/$pkgname.containerinfo") {
# package exists, why are we called? verify that it matches our expectations
return "$dir/$pkgname.tar" if is_wanted_dodcontainer($pool, $p, "$dir/$pkgname");
}
# we really need to download, handoff to ajax if not already done
BSHandoff::handoff(@$handoff) if $handoff && !$BSStdServer::isajax;

# download all missing blobs
my $path = $pool->pkg2path($p);
die("bad DoD container path: $path\n") unless $path =~ /^(.*)\?(.*?)$/;
my $regrepo = $1;
my @blobs = split(',', $2);
for my $blob (@blobs) {
next if -e "$dir/_blob.$blob";
my $tmp = "$dir/._blob.$blob.$$";
my $url = $repo->dodurl();
$url .= '/' unless $url =~ /\/$/;
$url .= "v2/$regrepo/blobs/$blob";
print "fetching: $url\n";
my $param = {'uri' => $url, 'filename' => $tmp, 'receiver' => \&BSHTTP::file_receiver, 'proxy' => $proxy};
$param->{'maxredirects'} = $maxredirects if defined $maxredirects;
my $r;
eval { $r = BSWatcher::rpc($param); };
if ($@) {
$@ =~ s/(\d* *)/$1$url: /;
die($@);
}
return unless defined $r;
if (!blob_matches_digest($tmp, $blob)) {
unlink($tmp);
die("$url: blob does not match digest\n");
}
rename($tmp, "$dir/_blob.$blob") || die("rename $tmp $dir/_blob.$blob: $!\n");
}

# delete old cruft
unlink("$dir/$pkgname.containerinfo");
unlink("$dir/$pkgname.obsbinlnk");

# write containerinfo file
my $data = $pool->pkg2data($p);
# hack: get tags from provides
my @tags;
for (@{$data->{'provides' || []}}) {
push @tags, $_ unless / = /;
}
push @tags, $data->{'name'} unless @tags;
my $mtime = time();
my @layers = @blobs;
shift @layers;
my $manifest = {
'Config' => $blobs[0],
'RepoTags' => \@tags,
'Layers' => \@layers,
};
my $manifest_ent = BSContar::create_manifest_entry($manifest, $mtime);
my $containerinfo = {
'tar_manifest' => $manifest_ent->{'data'},
'tar_size' => 1, # make construct_container_tar() happy
'tar_mtime' => $mtime,
'tar_blobids' => \@blobs,
'name' => $container,
'version' => $data->{'version'},
'tags' => \@tags,
'file' => "$pkgname.tar",
};
$containerinfo->{'release'} = $data->{'release'} if defined $data->{'release'};
my ($tar) = BSRepServer::Containertar::construct_container_tar($dir, $containerinfo);
($containerinfo->{'tar_md5sum'}, $containerinfo->{'tar_sha256sum'}, $containerinfo->{'tar_size'}) = BSContar::checksum_tar($tar);
BSRepServer::Containerinfo::writecontainerinfo("$dir/.$pkgname.containerinfo", "$dir/$pkgname.containerinfo", $containerinfo);

# write obsbinlnk file (do this last!)
my $lnk = BSRepServer::Containerinfo::containerinfo2nevra($containerinfo);
$lnk->{'source'} = $lnk->{'name'};
BSVerify::verify_nevraquery($lnk);
$lnk->{'hdrmd5'} = $containerinfo->{'tar_md5sum'};
$lnk->{'path'} = "$pkgname.tar";
BSUtil::store("$dir/.$pkgname.obsbinlnk", "$dir/$pkgname.obsbinlnk", $lnk);

return "$dir/$pkgname.tar";
}

sub fetchdodbinary {
my ($gdst, $pool, $repo, $p, $handoff) = @_;

die($repo->name()." is no dod repo\n") unless $repo->dodurl();
my $pkgname = $pool->pkg2name($p);
return fetchdodcontainer($gdst, $pool, $repo, $p, $handoff) if $pkgname =~ /^container:/;
my $path = $pool->pkg2path($p);
die("$path has an unsupported suffix\n") unless $path =~ /\.($binsufsre)$/;
my $suf = $1;
my $pkgname = $pool->pkg2name($p);
if (defined(&BSSolv::pool::pkg2inmodule) && $pool->pkg2inmodule($p)) {
$pkgname .= '-' . $pool->pkg2evr($p) . '.' . $pool->pkg2arch($p);
}
Expand Down
24 changes: 22 additions & 2 deletions src/backend/BSSched/DoD.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,12 @@ package BSSched::DoD;
use strict;
use warnings;

use Digest::MD5 ();

use BSUtil;
use BSSched::RPC;
use BSXML;
use BSHTTP;

=head2 gencookie - generate a cookie from file metadata
Expand Down Expand Up @@ -49,6 +54,7 @@ sub readparsed {
return 'baseurl missing in data' unless $baseurl;
my $moduleinfo = delete $data->{'/moduleinfo'};
for (values %$data) {
next unless ref($_) eq 'HASH';
$_->{'id'} = 'dod';
$_->{'hdrmd5'} = 'd0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0';
}
Expand Down Expand Up @@ -136,8 +142,21 @@ sub clean_obsolete_dodpackages {
}
my @nbins;
my $nbinsdirty;
my $clean_blobs;
while (@bins) {
my ($path, $id) = splice(@bins, 0, 2);
if ($path =~ s/\.obsbinlnk$//) {
$clean_blobs = 1;
if ($paths{"$path.tar"}) {
push @nbins, "$path.obsbinlnk", $id;
next;
}
$nbinsdirty = 1;
print " - :full/$path.tar [DoD cleanup]\n";
unlink("$dir/$path.obsbinlnk");
unlink("$dir/$path.containerinfo");
next;
}
if ($paths{$path}) {
push @nbins, $path, $id;
next;
Expand Down Expand Up @@ -193,15 +212,16 @@ sub dodcheck {
sub dodfetch_resume {
my ($ctx, $handle, $error) = @_;
my ($projid, $repoid, $arch) = split('/', $handle->{'_prpa'}, 3);
my $gctx = $ctx->{'gctx'};
if ($error) {
if (BSSched::RPC::is_transient_error($error)) {
$ctx->{'gctx'}->{'retryevents'}->addretryevent({'type' => 'repository', 'project' => $projid, 'repository' => $repoid, 'arch' => $arch});
$gctx->{'retryevents'}->addretryevent({'type' => 'repository', 'project' => $projid, 'repository' => $repoid, 'arch' => $arch});
}
return;
}
$ctx->setchanged($handle);
# drop cache
$ctx->{'gctx'}->{'repodatas'}->drop("$projid/$repoid", $arch);
$gctx->{'repodatas'}->drop("$projid/$repoid", $arch);
}

=head2 dodfetch - TODO: add summary
Expand Down
27 changes: 20 additions & 7 deletions src/backend/bs_repserver
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,14 +209,19 @@ sub getbinaryversions {
return unless defined $path;
# TODO: move it out of the loop otherwise the same files might be queried multiple times
my @s = stat($path);
$sizek = ($s[7] + 1023) >> 10;
$hdrmd5 = Build::queryhdrmd5($path);
if (!@s && $bin =~ /^container:/ && $path =~ /\.tar$/) {
$hdrmd5 = undef;
} else {
$sizek = ($s[7] + 1023) >> 10;
$hdrmd5 = Build::queryhdrmd5($path);
}
$needscan = 1;
}
if ($bin =~ /^container:/ && $path =~ /(\.tar(?:\..+)?)$/) {
my $n = "$bin$1";
my @s = stat($path);
@s = BSRepServer::Containertar::stat_container($path) if $1 eq '.tar' && !@s;
$hdrmd5 = $s[20] if $s[20] && !defined($hdrmd5);
push @res, {'name' => $n, 'hdrmd5' => $hdrmd5, 'sizek' => (($s[7] + 1023) >> 10)} if @s;
next;
}
Expand Down Expand Up @@ -1397,6 +1402,7 @@ sub getbinary_repository {
return if $BSStdServer::isajax && !defined $serial;
my $view = $cgi->{'view'} || '';
my $path = "$reporoot/$projid/$repoid/$arch/:full/$bin";
my $path_fd;
my $needscan;
if (! -f $path) {
# return by name
Expand All @@ -1423,11 +1429,18 @@ sub getbinary_repository {
}
undef $repo;
undef $pool;
if ($path =~ /\.tar$/ && ! -f $path && !$BSStdServer::isajax) {
return getbinary_info($cgi, $projid, $repoid, $arch, $path, $path) if $view eq 'fileinfo' || $view eq 'fileinfo_ext';
return undef if BSRepServer::Containertar::reply_container($path);
if (! -f $path) {
die("404 $bin: $!\n") unless $bin =~ /^container:/ && $path =~ /\.tar$/;;
if ($view ne 'fileinfo' && $view ne 'fileinfo_ext') {
if (!$BSStdServer::isajax) {
# use reply_container to directly send the blobs instead of constructing the complete tar
forwardevent($cgi, 'scanrepo', $projid, undef, $repoid, $arch) if $needscan;
BSRepServer::Containertar::reply_container($path);
return undef;
}
$path_fd = BSRepServer::Containertar::open_container($path);
}
}
die("404 $bin: $!\n") unless -f $path;
}
BSWatcher::serialize_end($serial) if defined $serial;
forwardevent($cgi, 'scanrepo', $projid, undef, $repoid, $arch) if $needscan;
Expand All @@ -1436,7 +1449,7 @@ sub getbinary_repository {
my $type = 'application/octet-stream';
$type = 'application/x-rpm' if $path=~ /\.rpm$/;
$type = 'application/x-debian-package' if $path=~ /\.deb$/;
BSWatcher::reply_file($path, "Content-Type: $type");
BSWatcher::reply_file($path_fd || $path, "Content-Type: $type");
return undef;
}

Expand Down

0 comments on commit e185c51

Please sign in to comment.