Skip to content

Commit

Permalink
Merge pull request #11185 from mlschroe/master
Browse files Browse the repository at this point in the history 
[backend] support DoD container registries
  • Loading branch information
@mlschroe
mlschroe committed May 27, 2021
2 parents b31e66a + fbeaccf commit 51ba1f4
Show file tree
Hide file tree
Showing 12 changed files with 807 additions and 36 deletions.
24 changes: 15 additions & 9 deletions src/backend/BSBearer.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,14 @@ use MIME::Base64;

use strict;

sub decode_reply {
my ($state, $json) = @_;
my $reply = JSON::XS::decode_json($json);
my $token = $reply->{'token'} || $reply->{'access_token'};
die("bearer auth rpc did not return a token\n") unless $token;
return $state->{'auth'} = "Bearer $token";
}

sub authenticator_function {
my ($state, $param, $wwwauthenticate) = @_;
return $state->{'auth'} if !$wwwauthenticate; # return last auth
Expand All @@ -37,24 +45,22 @@ sub authenticator_function {
my %auth = BSHTTP::parseauthenticate($wwwauthenticate);
if ($auth{'basic'} && defined($creds)) {
$auth = 'Basic '.MIME::Base64::encode_base64($creds, '');
$state->{'auth'} = $auth;
} elsif ($auth{'bearer'}) {
my $bearer = $auth{'bearer'};
my $realm = ($bearer->{'realm'} || [])->[0];
return undef unless $realm && $realm =~ /^https?:\/\//i;
return '' unless $realm && $realm =~ /^https?:\/\//i;
my @args = BSRPC::args($bearer, 'service', 'scope');
print "requesting bearer auth from $realm [@args]\n" if $state->{'verbose'};
my $bparam = { 'uri' => $realm };
push @{$bparam->{'headers'}}, 'Authorization: Basic '.MIME::Base64::encode_base64($creds, '') if defined($creds);
my $reply;
eval { $reply = BSRPC::rpc($bparam, \&JSON::XS::decode_json, @args); };
my $rpc = $state->{'rpccall'} || \&BSRPC::rpc;
my $decoder = sub {decode_reply($state, $_[0])};
eval { $auth = $rpc->($bparam, $decoder, @args) };
return undef unless defined $auth; # in progress
warn($@) if $@;
return undef unless $reply;
my $token = $reply->{'token'} || $reply->{'access_token'};
return undef unless $token;
$auth = "Bearer $token";
}
$state->{'auth'} = $auth if defined $auth;
return $auth;
return $auth || '';
}

sub generate_authenticator {
Expand Down
70 changes: 69 additions & 1 deletion src/backend/BSRepServer/DoD.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,13 @@
#
package BSRepServer::DoD;

use Digest::SHA ();

use BSWatcher ':https';
use BSVerify;
use BSHandoff;
use BSStdServer;
use BSUtil;

use Build;

Expand Down Expand Up @@ -51,14 +54,54 @@ sub is_wanted_dodbinary {
return 1;
}

sub is_wanted_dodcontainer {
my ($pool, $p, $path, $doverify) = @_;
my $q = BSUtil::retrieve("$path.obsbinlnk", 1);
return 0 unless $q;
my $data = $pool->pkg2data($p);
return 0 if $data->{'name'} ne $q->{'name'} || $data->{'version'} ne $q->{'version'};
BSVerify::verify_nevraquery($q) if $doverify; # just in case
return 1;
}

sub fetchdodcontainer {
my ($gdst, $pool, $repo, $p, $handoff) = @_;

my $pkgname = $pool->pkg2name($p);
$pkgname =~ s/^container://;
BSVerify::verify_filename($pkgname);
BSVerify::verify_simple($pkgname);
my $dir = "$gdst/:full";

if (-e "$dir/$pkgname.obsbinlnk" && -e "$dir/$pkgname.containerinfo") {
# package exists, why are we called? verify that it matches our expectations
return "$dir/$pkgname.tar" if is_wanted_dodcontainer($pool, $p, "$dir/$pkgname");
}
# we really need to download, handoff to ajax if not already done
BSHandoff::handoff(@$handoff) if $handoff && !$BSStdServer::isajax;

# download all missing blobs
my $path = $pool->pkg2path($p);
die("bad DoD container path: $path\n") unless $path =~ /^(.*)\?(.*?)$/;
my $regrepo = $1;
my @blobs = split(',', $2);
return undef unless BSRepServer::Registry::download_blobs($dir, $repo->dodurl(), $regrepo, \@blobs, $proxy, $maxredirects);

# write containerinfo and obsbinlnk files
my $data = $pool->pkg2data($p);
BSRepServer::Registry::construct_containerinfo($dir, $pkgname, $data, \@blobs);
return "$dir/$pkgname.tar";
}

sub fetchdodbinary {
my ($gdst, $pool, $repo, $p, $handoff) = @_;

die($repo->name()." is no dod repo\n") unless $repo->dodurl();
my $pkgname = $pool->pkg2name($p);
return fetchdodcontainer($gdst, $pool, $repo, $p, $handoff) if $pkgname =~ /^container:/;
my $path = $pool->pkg2path($p);
die("$path has an unsupported suffix\n") unless $path =~ /\.($binsufsre)$/;
my $suf = $1;
my $pkgname = $pool->pkg2name($p);
if (defined(&BSSolv::pool::pkg2inmodule) && $pool->pkg2inmodule($p)) {
$pkgname .= '-' . $pool->pkg2evr($p) . '.' . $pool->pkg2arch($p);
}
Expand Down Expand Up @@ -102,4 +145,29 @@ sub fetchdodbinary {
return $localname;
}

sub setmissingdodresources {
my ($gdst, $id, $dodresources) = @_;
my $dir = "$gdst/:full";
die("no doddata\n") unless -s "$dir/doddata";
my @dr = sort(BSUtil::unify(@$dodresources));
my $needed = BSUtil::retrieve("$dir/doddata.needed", 1);
return if $needed && BSUtil::identical($needed->{$id} || [], \@dr);
my $fd;
if (!BSUtil::lockopen($fd, '>>', "$dir/doddata.needed", 1)) {
warn("$dir/doddata.needed: $!\n");
return;
}
$needed = {};
$needed = BSUtil::retrieve("$dir/doddata.needed", 1) || {} if -s "$dir/doddata.needed";
if (!@dr) {
delete $needed->{$id};
delete $needed->{''}->{$id} if $needed->{''};
} else {
$needed->{$id} = \@dr;
$needed->{''}->{$id} = time();
}
BSUtil::store("$dir/.doddata.needed", "$dir/doddata.needed", $needed);
close($fd);
}

1;
152 changes: 151 additions & 1 deletion src/backend/BSRepServer/Registry.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,27 @@
package BSRepServer::Registry;

use JSON::XS ();
use Digest::SHA ();
use Time::Local ();

use BSConfiguration;
use BSWatcher ':https';
use BSTUF;
use BSUtil;
use BSConfiguration;
use BSVerify;
use BSBearer;
use BSContar;

use BSRepServer::Containertar;
use BSRepServer::Containerinfo;


use strict;

my $uploaddir = "$BSConfig::bsdir/upload";
my $blobdir = "$BSConfig::bsdir/blobs";

my %registry_authenticators;

sub select_manifest {
my ($mani, $goarch, $goos, $govariant) = @_;
Expand Down Expand Up @@ -64,4 +77,141 @@ sub extend_timestamp {
return $tuf;
}

sub blobstore_put {
my ($f, $dir) = @_;
return unless $f =~ /^_blob\.sha256:([0-9a-f]{3})([0-9a-f]{61})$/s;
my ($d, $b) = ($1, $2);
my @s = stat("$blobdir/sha256/$d/$b");
if (!@s) {
mkdir_p("$blobdir/sha256/$d") unless -d "$blobdir/sha256/$d";
return if link("$dir/$f", "$blobdir/sha256/$d/$b");
}
return unless link("$blobdir/sha256/$d/$b", "$blobdir/sha256/$d/$b.$$");
return if rename("$blobdir/sha256/$d/$b.$$", "$dir/$f");
unlink("$blobdir/sha256/$d/$b.$$");
}

sub blobstore_get {
my ($f, $dir) = @_;
return undef unless $f =~ /^_blob\.sha256:([0-9a-f]{3})([0-9a-f]{61})$/s;
my ($d, $b) = ($1, $2);
return link("$blobdir/sha256/$d/$b", "$dir/$f") ? 1 : undef;
}

sub blob_matches_digest {
my ($tmp, $digest) = @_;
my $ctx;
$ctx = Digest::SHA->new($1) if $digest =~ /^sha(256|512):/;
return 0 unless $ctx;
my $fd;
return 0 unless open ($fd, '<', $tmp);
$ctx->addfile($fd);
close($fd);
return (split(':', $digest, 2))[1] eq $ctx->hexdigest() ? 1 : 0;
}

sub doauthrpc {
my ($param, $xmlargs, @args) = @_;
$param = { %$param, 'resulthook' => sub { $xmlargs->($_[0]) } };
return BSWatcher::rpc($param, $xmlargs, @args);
}

sub download_blobs {
my ($dir, $url, $regrepo, $blobs, $proxy, $maxredirects) = @_;

$url .= '/' unless $url =~ /\/$/;
for my $blob (@$blobs) {
next if -e "$dir/_blob.$blob";
next if blobstore_get("_blob.$blob", $dir);
my $tmp = "$dir/._blob.$blob.$$";
my $authenticator = $registry_authenticators{"$url$regrepo"};
$authenticator = $registry_authenticators{"$url$regrepo"} = BSBearer::generate_authenticator(undef, 'verbose' => 1, 'rpccall' => \&doauthrpc) unless $authenticator;
my $bloburl = "${url}v2/$regrepo/blobs/$blob";
# print "fetching: $bloburl\n";
my $param = {'uri' => $bloburl, 'filename' => $tmp, 'receiver' => \&BSHTTP::file_receiver, 'proxy' => $proxy};
$param->{'authenticator'} = $authenticator;
$param->{'maxredirects'} = $maxredirects if defined $maxredirects;
my $r;
eval { $r = BSWatcher::rpc($param); };
if ($@) {
unlink($tmp);
$@ =~ s/(\d* *)/$1$bloburl: /;
die($@);
}
return unless defined $r;
if (!blob_matches_digest($tmp, $blob)) {
unlink($tmp);
die("$bloburl: blob does not match digest\n");
}
rename($tmp, "$dir/_blob.$blob") || die("rename $tmp $dir/_blob.$blob: $!\n");
blobstore_put("_blob.$blob", $dir);
}
return 1;
}

sub construct_containerinfo {
my ($dir, $pkgname, $data, $blobs) = @_;

BSVerify::verify_filename($pkgname);
BSVerify::verify_simple($pkgname);

# delete old cruft
unlink("$dir/$pkgname.containerinfo");
unlink("$dir/$pkgname.obsbinlnk");

# try to get a timestamp from the config blob for reproducibility
my $mtime = time();
if (-e "$dir/_blob.$blobs->[0]" && -s _ < 65536) {
my $configjson = readstr("$dir/_blob.$blobs->[0]", 1) || '';
if ($configjson =~ /\"created\"\s*?:\s*?\"([123]\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)/) {
my $t = eval { Time::Local::timegm($6, $5, $4, $3, $2 - 1, $1) };
warn($@) if $@;
$mtime = $t if $t && $t < $mtime;
}
}

# hack: get tags from provides
my @tags;
for (@{$data->{'provides' || []}}) {
push @tags, $_ unless / = /;
}
s/^container:// for @tags;
push @tags, $data->{'name'} unless @tags;
my @layers = @$blobs;
shift @layers;
my $manifest = {
'Config' => $blobs->[0],
'RepoTags' => \@tags,
'Layers' => \@layers,
};
my $manifest_ent = BSContar::create_manifest_entry($manifest, $mtime);
my $containerinfo = {
'tar_manifest' => $manifest_ent->{'data'},
'tar_size' => 1, # make construct_container_tar() happy
'tar_mtime' => $mtime,
'tar_blobids' => $blobs,
'name' => $pkgname,
'version' => $data->{'version'},
'tags' => \@tags,
'file' => "$pkgname.tar",
};
$containerinfo->{'release'} = $data->{'release'} if defined $data->{'release'};
my ($tar) = BSRepServer::Containertar::construct_container_tar($dir, $containerinfo);
($containerinfo->{'tar_md5sum'}, $containerinfo->{'tar_sha256sum'}, $containerinfo->{'tar_size'}) = BSContar::checksum_tar($tar);
BSRepServer::Containerinfo::writecontainerinfo("$dir/.$pkgname.containerinfo", "$dir/$pkgname.containerinfo", $containerinfo);

# write obsbinlnk file (do this last!)
my $lnk = BSRepServer::Containerinfo::containerinfo2nevra($containerinfo);
$lnk->{'source'} = $lnk->{'name'};
# add self-provides
push @{$lnk->{'provides'}}, "$lnk->{'name'} = $lnk->{'version'}";
for my $tag (@{$containerinfo->{tags}}) {
push @{$lnk->{'provides'}}, "container:$tag" unless "container:$tag" eq $lnk->{'name'};
}
BSVerify::verify_nevraquery($lnk);
$lnk->{'hdrmd5'} = $containerinfo->{'tar_md5sum'};
$lnk->{'path'} = "$pkgname.tar";
BSUtil::store("$dir/.$pkgname.obsbinlnk", "$dir/$pkgname.obsbinlnk", $lnk);
}

1;
2 changes: 2 additions & 0 deletions src/backend/BSSched/BuildJob/Docker.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,8 @@ sub check {
if ($state eq 'scheduled') {
my $dods = BSSched::DoD::dodcheck($ctx, $pool, $myarch, @edeps);
return ('blocked', $dods) if $dods;
$dods = BSSched::DoD::dodcheck($ctx, $ctx->{'pool'}, $myarch, map {$_->{'name'}} @cbdep);
return ('blocked', $dods) if $dods;
}
return ($state, $data);
}
Expand Down
2 changes: 2 additions & 0 deletions src/backend/BSSched/BuildJob/KiwiImage.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,8 @@ sub check {
if ($state eq 'scheduled') {
my $dods = BSSched::DoD::dodcheck($ctx, $pool, $myarch, @edeps);
return ('blocked', $dods) if $dods;
$dods = BSSched::DoD::dodcheck($ctx, $ctx->{'pool'}, $myarch, $cbdep->{'name'}) if $cbdep;
return ('blocked', $dods) if $dods;
}
return ($state, $data);
}
Expand Down
4 changes: 2 additions & 2 deletions src/backend/BSSched/BuildRepo.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -957,7 +957,7 @@ sub addrepo_scan {
my $doddata;
if ($BSConfig::enable_download_on_demand) {
$doddata = BSSched::DoD::get_doddata($gctx, $prp, $arch);
($dirty, $r) = BSSched::DoD::put_doddata_in_cache($pool, $prp, $r, $doddata, $dir);
($dirty, $r) = BSSched::DoD::put_doddata_in_cache($gctx, $doddata, $pool, $prp, $r, $dir);
}

my @bins;
Expand Down Expand Up @@ -994,7 +994,7 @@ sub addrepo_scan {
return undef unless $r;
# write solv file (unless alien arch)
if ($dirty && $arch eq $gctx->{'arch'}) {
@bins = BSSched::DoD::clean_obsolete_dodpackages($pool, $r, $dir, @bins) if $doddata;
@bins = BSSched::DoD::clean_obsolete_dodpackages($gctx, $doddata, $pool, $prp, $r, $dir, @bins) if $doddata;
writesolv("$dir.solv.new", "$dir.solv", $r);
}
$repocache->setcache($prp, $arch) if $repocache;
Expand Down

0 comments on commit 51ba1f4

Please sign in to comment.