Skip to content

Commit

Permalink
[backend] switch helm storage from .helm.tar to a .helminfo file
Browse files Browse the repository at this point in the history 
This allows us to publish helm charts that are not stored into
a registry.
  • Loading branch information
@mlschroe
mlschroe committed Jul 31, 2020
1 parent 46f22f6 commit 08a8f09
Show file tree
Hide file tree
Showing 6 changed files with 73 additions and 86 deletions.
50 changes: 21 additions & 29 deletions src/backend/BSContar.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,40 +298,32 @@ sub create_dist_manifest_list {
}

sub container_from_helm {
my ($tarfd) = @_;
my @tarstat = stat($tarfd);
die("stat: $!\n") unless @tarstat;
my $mtime = $tarstat[9];
my $tar = BSTar::list($tarfd);
$_->{'file'} = $tarfd for @$tar;
my %tar = map {$_->{'name'} => $_} @$tar;
# read helm manifest
my $helmmanifest_ent = $tar{'manifest.json'};
die("helm.tar file has no manifest.json\n") unless $helmmanifest_ent;
my $helmmanifest_json = BSTar::extract($helmmanifest_ent->{'file'}, $helmmanifest_ent);
my $helmmanifest = JSON::XS::decode_json($helmmanifest_json);
my $chart = $helmmanifest->{'chart'};
die("helm manifest does not contain a chart\n") unless $chart && ref($chart) eq '';
my $repotags = $helmmanifest->{'tags'};
$repotags = undef unless $repotags && ref($repotags) eq 'ARRAY';
die("helm.tar file has no config.json\n") unless $tar{'config.json'};
die("helm.tar file has no $chart\n") unless $tar{$chart};
# replace helm manifest with fake container manifest
my ($chartfile, $config_json, $repotags) = @_;
my $fd;
die("$chartfile: $!\n") unless open($fd, '<', $chartfile);
my @s = stat($fd);
die("stat: $!\n") unless @s;
my $mtime = $s[9];
# create ent for the chart
my $chartbasename = $chartfile;
$chartbasename =~ s/^.*\///;
my $chart_ent = { 'name' => $chartbasename, 'offset' => 0, 'size' => $s[7], 'mtime' => $mtime, 'file' => $fd };
if ($chartbasename =~ /($?:\.tar\.gz|\.tgz)$/) {
$chart_ent->{'mimetype'} = 'application/tar+gzip';
} else {
$chart_ent->{'mimetype'} = 'application/tar';
}
# create ent for the config
my $config_ent = { 'name' => 'config.json', 'size' => length($config_json), 'data' => $config_json, 'mtime' => $mtime };
$config_ent->{'mimetype'} = 'application/vnd.cncf.helm.config.v1+json';
# create ent for the manifest
my $manifest = {
'Layers' => [ $chart ],
'Layers' => [ $chartbasename ],
'Config' => 'config.json',
'RepoTags' => $repotags || [],
};
my $manifest_ent = create_manifest_entry($manifest, $mtime);
%{$tar{'manifest.json'}} = %$manifest_ent;
# set mime types
$tar{'config.json'}->{'mimetype'} = 'application/vnd.cncf.helm.config.v1+json';
if ($chart =~ /($?:\.tar\.gz|\.tgz)$/) {
$tar{$chart}->{'mimetype'} = 'application/tar+gzip';
} else {
$tar{$chart}->{'mimetype'} = 'application/tar';
}
return ($tar, $mtime);
return ([ $manifest_ent, $config_ent, $chart_ent ], $mtime);
}

1;
9 changes: 9 additions & 0 deletions src/backend/BSPublisher/Container.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,13 @@ sub upload_to_registry {
die("need a blobdir for containerinfo uploads\n") unless $blobdir;
push @uploadfiles, "$blobdir/container.".scalar(@uploadfiles).".containerinfo";
BSRepServer::Containerinfo::writecontainerinfo($uploadfiles[-1], undef, $containerinfo);
} elsif ($file =~ /(.*)\.tgz$/ && ($containerinfo->{'type'} || '') eq 'helm') {
my $helminfofile = "$1.helminfo";
$blobdir = $containerinfo->{'blobdir'};
die("need a blobdir for helminfo uploads\n") unless $blobdir;
die("bad publishfile\n") unless $helminfofile =~ /^\Q$blobdir\E\//; # just in case
push @uploadfiles, $helminfofile;
BSRepServer::Containerinfo::writecontainerinfo($uploadfiles[-1], undef, $containerinfo);
} elsif ($file =~ /\.tar$/) {
push @uploadfiles, $file;
} else {
Expand Down Expand Up @@ -534,6 +541,8 @@ sub do_local_uploads {
die("need a blobdir for containerinfo uploads\n") unless $containerinfo->{'blobdir'};
} elsif ($file =~ /\.tar$/) {
$containerinfo->{'uploadfile'} = $file;
} elsif ($file =~ /\.tgz$/ && ($containerinfo->{'type'} || '') eq 'helm') {
$containerinfo->{'uploadfile'} = $file;
} else {
my $tmpfile = decompress_container($file);
$containerinfo->{'uploadfile'} = $tmpfile;
Expand Down
60 changes: 22 additions & 38 deletions src/backend/BSPublisher/Helm.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,51 +23,35 @@
package BSPublisher::Helm;

use BSTar;
use BSUtil;

use strict;

sub readcontainerinfo {
my ($dir, $helmtar) = @_;
my $h;
return undef unless open($h, '<', "$dir/$helmtar");
my $tar = BSTar::list($h);
return undef unless $tar;
my %tar = map {$_->{'name'} => $_} @$tar;
my $manifest_ent = $tar{'manifest.json'};
return undef unless $manifest_ent && $manifest_ent->{'size'} < 100000;
my $manifest_json = BSTar::extract($h, $manifest_ent);
my $manifest = JSON::XS::decode_json($manifest_json);
close $h;

my $d;
eval { $d = JSON::XS::decode_json($manifest_json) };
# this also works as a containerinfo substitute
sub readhelminfo {
my ($dir, $helminfofile) = @_;
return undef unless -e "$dir/$helminfofile";
return undef unless (-s _) < 1000000;
my $m = readstr("$dir/$helminfofile");
my $d;
eval { $d = JSON::XS::decode_json($m); };
return undef unless $d && ref($d) eq 'HASH';
my $tags = $d->{'tags'};
$tags = [] unless $tags && ref($tags) eq 'ARRAY';
for (@$tags) {
$_ = undef unless defined($_) && ref($_) eq '';
}
@$tags = grep {defined($_)} @$tags;

my $name = $d->{'name'};
$name = undef unless defined($name) && ref($name) eq '';
if (!defined($name) && @$tags) {
# no name specified, get it from first tag
$name = $tags->[0];
$name =~ s/[:\/]/-/g;
if (exists $d->{'tags'}) {
$d->{'tags'} = [] unless ref($d->{'tags'}) eq 'ARRAY';
for (splice @{$d->{'tags'}}) {
push @{$d->{'tags'}}, $_ if defined($_) && ref($_) eq '';
}
}

my $containerinfo = { 'type' => 'helm' };
$containerinfo->{'name'} = $name if defined $name;
$containerinfo->{'tags'} = $tags if @$tags;
for my $k (qw{chart disturl buildtime version release}) {
for my $k (qw{disturl buildtime name version release config_json}) {
my $v = $d->{$k};
$containerinfo->{$k} = $v if defined($v) && ref($v) eq '';
$d->{$k} = $v if defined($v) && ref($v) eq '';
}

return undef unless $containerinfo->{'chart'} && $tar{$containerinfo->{'chart'}};

return $containerinfo;
return undef unless $d->{'name'} && $d->{'config_json'};
$d->{'chart'} = $helminfofile;
$d->{'chart'} =~ s/.*\///;
$d->{'chart'} =~ s/\.helminfo$/.tgz/;
$d->{'type'} = 'helm';
return $d;
}

1;
4 changes: 2 additions & 2 deletions src/backend/BSPublisher/Registry.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -475,8 +475,8 @@ sub push_containers {
my $tarfd;
if ($containerinfo->{'uploadfile'}) {
open($tarfd, '<', $containerinfo->{'uploadfile'}) || die("$containerinfo->{'uploadfile'}: $!\n");
if ($containerinfo->{'type'} && $containerinfo->{'type'} eq 'helm') {
($tar, $mtime) = BSContar::container_from_helm($tarfd);
if (($containerinfo->{'type'} || '') eq 'helm') {
($tar, $mtime) = BSContar::container_from_helm($containerinfo->{'uploadfile'}, $containerinfo->{'config_json'}, $containerinfo->{'tags'});
} else {
($tar, $mtime) = BSContar::normalize_container($tarfd, 1);
}
Expand Down
26 changes: 12 additions & 14 deletions src/backend/bs_publish
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1522,6 +1522,13 @@ sub sync_to_stage {
qsystem('rsync', '-a', @rsync_extra_options, '--timeout', '120', "$extrepodir_sync/$filename", $BSConfig::stageserver_sync."/".$filename) && warn(" trigger rsync failed at ".localtime(time).": $?\n");
}

sub create_blobdir {
my $blobdir = "$uploaddir/publisher.$$.blobs";
BSUtil::cleandir($blobdir);
mkdir_p($blobdir);
return $blobdir;
}

sub clean_blobdir {
my ($blobdir) = @_;
return unless $blobdir;
Expand Down Expand Up @@ -1981,11 +1988,7 @@ sub publish {
}
if ($bin =~ /^_blob\.sha256:[0-9a-f]{64}$/) {
# container blobid, collect
if (!$blobdir) {
$blobdir = "$uploaddir/publisher.$$.blobs";
BSUtil::cleandir($blobdir);
mkdir_p($blobdir);
}
$blobdir ||= create_blobdir();
if (! -e "$blobdir/$bin") {
link("$r/$bin", "$blobdir/$bin") || die("link $r/$bin: $blobdir/$bin$!\n");
}
Expand Down Expand Up @@ -2038,13 +2041,12 @@ sub publish {
$p = $bin;
$p = mapsleimage($sleimagedata, "$reporoot/$prp/$arch/:repo", $rbin, $p) if $sleimagedata;
$kiwimedium = "$arch/$1" if !$2 && -e "$r/$1.packages";
} elsif ($bin =~ /\.helm\.tar$/) {
} elsif ($bin =~ /(.*)\.tgz$/ && -e "$r/$1.helminfo") {
my @s = stat("$r/$rbin");
next unless @s;
eval { $containerinfo = BSPublisher::Helm::readcontainerinfo($r, $rbin) };
warn("$@") if $@;
eval { $containerinfo = BSPublisher::Helm::readhelminfo($r, "$1.helminfo") };
if ($containerinfo) {
$p = $multicontainer ? "$arch/$bin" : $bin;
$p = $bin;
$containerinfo->{'arch'} = $arch;
$containerinfo->{'_id'} = "$s[9]/$s[7]/$s[1]";
$containerinfo->{'_origin'} = $repoinfo->{'binaryorigins'}->{$rbin} if defined $repoinfo->{'binaryorigins'}->{$rbin};
Expand Down Expand Up @@ -2258,11 +2260,7 @@ sub publish {
my $containerinfo = $containers{$p};
if ($containerinfo->{'publishfile'}) {
# link container into the blobdir
if (!$blobdir) {
$blobdir = "$uploaddir/publisher.$$.blobs";
BSUtil::cleandir($blobdir);
mkdir_p($blobdir);
}
$blobdir ||= create_blobdir();
my $tmpfile = "$blobdir/$containerinfo->{'arch'}:$pp";
link($bins{$p}, $tmpfile) || die("link $bins{$p} $tmpfile: $!\n");
$containerinfo->{'publishfile'} = $tmpfile;
Expand Down
10 changes: 7 additions & 3 deletions src/backend/bs_regpush
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,9 +391,13 @@ sub open_tarfile {
my $containerinfo = JSON::XS::decode_json($containerinfo_json);
$govariant = $containerinfo->{'govariant'};
$tar = construct_container_tar($containerinfo);
} elsif ($tarfile =~ /\.helm.tar$/) {
open($tarfd, '<', $tarfile) || die("$tarfile: $!\n");
($tar) = BSContar::container_from_helm($tarfd);
} elsif ($tarfile =~ /\.helminfo$/) {
my $chart = $tarfile;
$chart =~ s/\.helminfo$/.tgz/;
die("$chart: $!\n") unless -e $chart;
my $helminfo_json = readstr($tarfile);
my $helminfo = JSON::XS::decode_json($helminfo_json);
($tar) = BSContar::container_from_helm($chart, $helminfo->{'config_json'}, $helminfo->{'tags'});
} else {
open($tarfd, '<', $tarfile) || die("$tarfile: $!\n");
$tar = BSTar::list($tarfd);
Expand Down

0 comments on commit 08a8f09

Please sign in to comment.