Skip to content

Commit

Permalink
Merge pull request #15572 from mlschroe/master
Browse files Browse the repository at this point in the history 
 [backend] implement artifacthub data publishing
  • Loading branch information
@mlschroe
mlschroe committed Jan 31, 2024
2 parents 194c713 + 271ef23 commit 0dbe62d
Show file tree
Hide file tree
Showing 5 changed files with 127 additions and 31 deletions.
57 changes: 56 additions & 1 deletion src/backend/BSContar.pm
View file
Expand Up @@ -26,6 +26,7 @@ use JSON::XS ();
use Digest::SHA ();
use Digest::MD5 ();
use Compress::Zlib ();
use Scalar::Util;
use POSIX;

use BSUtil;
Expand All @@ -44,6 +45,8 @@ our $mt_oci_config = 'application/vnd.oci.image.config.v1+json';
our $mt_oci_layer_gzip = 'application/vnd.oci.image.layer.v1.tar+gzip';
our $mt_oci_layer_zstd = 'application/vnd.oci.image.layer.v1.tar+zstd';
our $mt_helm_config = 'application/vnd.cncf.helm.config.v1+json';
our $mt_artifacthub_config = 'application/vnd.cncf.artifacthub.config.v1+yaml';
our $mt_artifacthub_layer = 'application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml';

sub blobid {
return 'sha256:'.Digest::SHA::sha256_hex($_[0]);
Expand Down Expand Up @@ -270,6 +273,7 @@ sub get_config {
die("File $config_file not included in tar\n") unless $config_ent;
my $config_json = BSTar::extract($config_ent->{'file'}, $config_ent);
$config_ent->{'blobid'} ||= blobid($config_json); # convenience
return ($config_ent, {}) if $config_json eq ''; # workaround for artifacthub
my $config = JSON::XS::decode_json($config_json);
return ($config_ent, $config);
}
Expand Down Expand Up @@ -442,6 +446,57 @@ sub container_from_helm {
return ($tar, $mtime, \@layercomp);
}

sub unparse_yaml_string {
my ($d) = @_;
return "''" unless length $d;
return "\"$d\"" if Scalar::Util::looks_like_number($d);
if ($d =~ /[\x00-\x1f\x7f-\x9f\']/) {
$d =~ s/\\/\\\\/g;
$d =~ s/\"/\\\"/g;
$d =~ s/([\x00-\x1f\x7f-\x9f])/'\x'.sprintf("%X",ord($1))/ge;
return "\"$d\"";
} elsif ($d =~ /^[\!\&*{}[]|>@`"'#%, ]/s) {
return "'$d'";
} elsif ($d =~ /: / || $d =~ / #/ || $d =~ /[: \t]\z/) {
return "'$d'";
} elsif ($d eq '~' || $d eq 'null' || $d eq 'true' || $d eq 'false' && $d =~ /^(?:---|\.\.\.)/s) {
return "'$d'";
} elsif ($d =~ /^[-?:](?:\s|\z)/s) {
return "'$d'";
} else {
return $d;
}
}

sub create_artifacthub_yaml {
my ($artifacthubdata) = @_;
my ($repoid, $name, $email) = split(':', $artifacthubdata, 3);
my $yaml = '';
$yaml .= "repositoryID: ".unparse_yaml_string($repoid)."\n" if $repoid;
my $owners = '';
$owners .= " name: ".unparse_yaml_string($name)."\n" if $name;
$owners .= " email: ".unparse_yaml_string($email)."\n" if $email;
$owners =~ s/^ / -/;
$yaml .= "owners:\n$owners" if $owners;
return $yaml;
}

sub container_from_artifacthub {
my ($artifacthubdata, $mtime) = @_;
my $artifacthub_yaml = create_artifacthub_yaml($artifacthubdata);
my $config_ent = { 'name' => 'config.yaml', 'mtime' => $mtime, 'data' => '', 'size' => 0, 'mimetype' => $mt_artifacthub_config };
my $layer_ent = { 'name' => 'artifacthub-repo.yml', 'mtime' => $mtime, 'data' => $artifacthub_yaml, 'size' => length($artifacthub_yaml), 'mimetype' => $mt_artifacthub_layer };
$layer_ent->{'annotations'}->{'org.opencontainers.image.title'} = 'artifacthub-repo.yml';
my $manifest = {
'Layers' => [ 'artifacthub-repo.yml' ],
'Config' => 'config.yaml',
'RepoTags' => [ 'artifacthub.io' ],
};
my $manifest_ent = create_manifest_entry($manifest, $mtime);
my $tar = [ $manifest_ent, $config_ent, $layer_ent ];
return ($tar, $mtime, [ '' ]);
}

sub create_config_data {
my ($config_ent, $oci) = @_;
my $config_data = {
Expand Down Expand Up @@ -482,7 +537,7 @@ sub create_layer_data {
'size' => 0 + $layer_ent->{'size'},
'digest' => $layer_ent->{'blobid'} || blobid_entry($layer_ent),
};
$layer_data->{'annotations'} = { %$annotations } if $annotations;
$layer_data->{'annotations'} = { %{$layer_ent->{'annotations'} || {}}, %{$annotations || {}} } if $layer_ent->{'annotations'} || $annotations;
if ($comp eq 'zstd' && $lcomp && $lcomp =~ /^zstd:chunked/) {
my @c = split(',', $lcomp);
$layer_data->{'annotations'}->{'io.github.containers.zstd-chunked.manifest-position'} = $c[1] if $c[1];
Expand Down
39 changes: 33 additions & 6 deletions src/backend/BSPublisher/Container.pm
View file
Expand Up @@ -199,7 +199,7 @@ sub upload_all_containers {
for my $p (sort keys %$containers) {
my $containerinfo = $containers->{$p};
my $arch = $containerinfo->{'arch'};
my $goarch = $containerinfo->{'goarch'} || (($containerinfo->{'type'} || '') eq 'helm' ? 'any' : $arch);
my $goarch = $containerinfo->{'goarch'} || (($containerinfo->{'type'} || '') eq 'helm' || ($containerinfo->{'type'} || '') eq 'artifacthub' ? 'any' : $arch);
$goarch .= ":$containerinfo->{'govariant'}" if $containerinfo->{'govariant'};
$goarch .= "_$containerinfo->{'goos'}" if $containerinfo->{'goos'} && $containerinfo->{'goos'} ne 'linux';
my @tags = $mapper->($registry, $containerinfo, $projid, $repoid, $arch);
Expand Down Expand Up @@ -308,11 +308,12 @@ sub reconstruct_container {
BSTar::writetarfile($dst, $dstfinal, $tar, 'mtime' => $mtime) if $tar;
}

sub create_container_dist_info {
my ($containerinfo, $oci, $platforms) = @_;
my $file = $containerinfo->{'publishfile'};
sub open_container_tar {
my ($containerinfo, $file) = @_;
my ($tar, $mtime, $layer_compression);
if (!defined($file)) {
if (($containerinfo->{'type'} || '') eq 'artifacthub') {
($tar, $mtime, $layer_compression) = BSContar::container_from_artifacthub($containerinfo->{'artifacthubdata'});
} elsif (!defined($file)) {
($tar, $mtime, $layer_compression) = BSPublisher::Containerinfo::construct_container_tar($containerinfo, 1);
} elsif (($containerinfo->{'type'} || '') eq 'helm') {
($tar, $mtime, $layer_compression) = BSContar::container_from_helm($file, $containerinfo->{'config_json'}, $containerinfo->{'tags'});
Expand All @@ -330,6 +331,12 @@ sub create_container_dist_info {
$_->{'file'} = $tarfd for @$tar;
}
die("incomplete containerinfo\n") unless $tar;
return ($tar, $mtime, $layer_compression);
}

sub create_container_dist_info {
my ($containerinfo, $oci, $platforms) = @_;
my ($tar, $mtime, $layer_compression) = open_container_tar($containerinfo, $containerinfo->{'publishfile'});
my %tar = map {$_->{'name'} => $_} @$tar;
my ($manifest_ent, $manifest) = BSContar::get_manifest(\%tar);
my ($config_ent, $config) = BSContar::get_config(\%tar, $manifest);
Expand Down Expand Up @@ -529,9 +536,10 @@ sub upload_to_registry {
my $multiarch = 0; # XXX: use $data->{'multiarch'}
$multiarch = 1 if @$containerinfos > 1;
$multiarch = 0 if @$containerinfos == 1 && ($containerinfos->[0]->{'type'} || '') eq 'helm';
$multiarch = 0 if @$containerinfos == 1 && ($containerinfos->[0]->{'type'} || '') eq 'artifacthub';
my $oci;
for my $containerinfo (@$containerinfos) {
$oci = 1 if ($containerinfo->{'type'} || '') eq 'helm';
$oci = 1 if ($containerinfo->{'type'} || '') eq 'helm' || ($containerinfo->{'type'} || '') eq 'artifacthub';
$oci = 1 if grep {$_ && $_ ne 'gzip'} @{$containerinfo->{'layer_compression'} || []};
}

Expand Down Expand Up @@ -569,6 +577,10 @@ sub upload_to_registry {
my $file = $containerinfo->{'publishfile'};
my $wrote_containerinfo;
if (!defined($file)) {
if (($containerinfo->{'type'} || '') eq 'artifacthub') {
push @uploadfiles, "artifacthub:$containerinfo->{'artifacthubdata'}";
next;
}
# tar file needs to be constructed from blobs
$blobdir = $containerinfo->{'blobdir'};
die("need a blobdir for containerinfo uploads\n") unless $blobdir;
Expand Down Expand Up @@ -832,6 +844,13 @@ sub do_local_uploads {
push @{$todo{$tag}}, $containerinfo;
}
}
my $gun = $registry->{'notary_gunprefix'} || $registry->{'server'};
$gun =~ s/^https?:\/\///;
$gun = '' if $gun eq 'local:';
if (($data->{'artifacthubdata'} || {})->{"$gun/$repository"} && !$uptags->{'artifacthub.io'}) {
my $containerinfo = { 'type' => 'artifacthub', 'artifacthubdata' => $data->{'artifacthubdata'}->{"$gun/$repository"} };
push @{$todo{'artifacthub.io'}}, $containerinfo;
}
eval {
BSPublisher::Registry::push_containers($registry, $projid, $repoid, $repository, \%todo, $data);
};
Expand Down Expand Up @@ -877,6 +896,14 @@ sub do_remote_uploads {
my $digests = upload_to_registry($registry, $projid, $repoid, $repository, \@containerinfos, \@tags, $data, $repostate);
$containerdigests .= $digests;
}
my $gun = $registry->{'notary_gunprefix'} || $registry->{'server'};
$gun =~ s/^https?:\/\///;
$gun = '' if $gun eq 'local:';
if (($data->{'artifacthubdata'} || {})->{"$gun/$repository"} && !$uptags->{'artifacthub.io'}) {
my $containerinfo = { 'type' => 'artifacthub', 'artifacthubdata' => $data->{'artifacthubdata'}->{"$gun/$repository"} };
my $digests = upload_to_registry($registry, $projid, $repoid, $repository, [ $containerinfo ], [ 'artifacthub.io' ], $data, $repostate);
$containerdigests .= $digests;
}
# all is pushed, now clean the rest
add_notary_upload($notary_uploads, $registry, $repository, $containerdigests);
delete_obsolete_tags_from_registry($registry, $repository, $containerdigests, $repostate);
Expand Down
44 changes: 25 additions & 19 deletions src/backend/BSPublisher/Registry.pm
View file
Expand Up @@ -548,6 +548,28 @@ sub create_manifestinfo {
push_manifestinfo($repodir, $imginfo->{'distmanifest'}, JSON::XS->new->utf8->canonical->encode($imginfo));
}

sub open_container_tar {
my ($containerinfo, $file) = @_;
my ($tar, $mtime, $layer_compression);
if (($containerinfo->{'type'} || '') eq 'artifacthub') {
($tar, $mtime, $layer_compression) = BSContar::container_from_artifacthub($containerinfo->{'artifacthubdata'});
} elsif (!defined($file)) {
($tar, $mtime, $layer_compression) = BSPublisher::Containerinfo::construct_container_tar($containerinfo, 1);
# set blobfile in entries so we can create a link in push_blob
for (@$tar) {
$_->{'blobfile'} = "$containerinfo->{'blobdir'}/_blob.$_->{'blobid'}" if $_->{'blobid'};
}
} elsif (($containerinfo->{'type'} || '') eq 'helm') {
($tar, $mtime, $layer_compression) = BSContar::container_from_helm($file, $containerinfo->{'config_json'}, $containerinfo->{'tags'});
} else {
my $tarfd;
open($tarfd, '<', $file) || die("$file: $!\n");
($tar, $mtime, undef, undef, $layer_compression) = BSContar::normalize_container($tarfd, 1);
}
die("incomplete containerinfo\n") unless $tar;
return ($tar, $mtime, $layer_compression);
}

sub push_containers {
my ($registry, $projid, $repoid, $repo, $tags, $data) = @_;

Expand Down Expand Up @@ -594,14 +616,15 @@ sub push_containers {
my $multiarch = $data->{'multiarch'};
$multiarch = 1 if @$containerinfos > 1;
$multiarch = 0 if @$containerinfos == 1 && ($containerinfos->[0]->{'type'} || '') eq 'helm';
$multiarch = 0 if @$containerinfos == 1 && ($containerinfos->[0]->{'type'} || '') eq 'artifacthub';
die("must use multiarch if multiple containers are to be pushed\n") if @$containerinfos > 1 && !$multiarch;
my %multiplatforms;
my @multimanifests;
my @imginfos;
my $oci;
# use oci types if we have a helm chart or we use a nonstandard compression
for my $containerinfo (@$containerinfos) {
$oci = 1 if ($containerinfo->{'type'} || '') eq 'helm';
$oci = 1 if ($containerinfo->{'type'} || '') eq 'helm' || ($containerinfo->{'type'} || '') eq 'artifacthub';
$oci = 1 if grep {$_ && $_ ne 'gzip'} @{$containerinfo->{'layer_compression'} || []};
}
for my $containerinfo (@$containerinfos) {
Expand All @@ -620,22 +643,7 @@ sub push_containers {
next;
}

my ($tar, $mtime, $layer_compression);
my $tarfd;
if ($containerinfo->{'uploadfile'}) {
open($tarfd, '<', $containerinfo->{'uploadfile'}) || die("$containerinfo->{'uploadfile'}: $!\n");
if (($containerinfo->{'type'} || '') eq 'helm') {
($tar, $mtime, $layer_compression) = BSContar::container_from_helm($containerinfo->{'uploadfile'}, $containerinfo->{'config_json'}, $containerinfo->{'tags'});
} else {
($tar, $mtime, undef, undef, $layer_compression) = BSContar::normalize_container($tarfd, 1);
}
} else {
($tar, $mtime, $layer_compression) = BSPublisher::Containerinfo::construct_container_tar($containerinfo, 1);
# set blobfile in entries so we can create a link in push_blob
for (@$tar) {
$_->{'blobfile'} = "$containerinfo->{'blobdir'}/_blob.$_->{'blobid'}" if $_->{'blobid'};
}
}
my ($tar, $mtime, $layer_compression) = open_container_tar($containerinfo, $containerinfo->{'uploadfile'});
my %tar = map {$_->{'name'} => $_} @$tar;

my ($manifest_ent, $manifest) = BSContar::get_manifest(\%tar);
Expand All @@ -657,7 +665,6 @@ sub push_containers {
$platformstr .= " variant:$govariant" if $govariant;
if ($multiplatforms{$platformstr}) {
print "ignoring $containerinfo->{'file'}, already have $platformstr\n";
close $tarfd if $tarfd;
next;
}
$multiplatforms{$platformstr} = 1;
Expand Down Expand Up @@ -690,7 +697,6 @@ sub push_containers {
push_blob($repodir, $layer_ent);
$knownblobs{$layer_blobid} = 1;
}
close $tarfd if $tarfd;

# put manifest into repo
my $mani = BSContar::create_dist_manifest_data($config_data, \@layer_data, $oci);
Expand Down
5 changes: 5 additions & 0 deletions src/backend/bs_publish
View file
Expand Up @@ -2857,6 +2857,11 @@ sub publish {
'publishid' => $publishid,
'multiarch' => $multicontainer,
};
if ($config->{'publishflags:artifacthub'}) {
for (@{$config->{'publishflags'} || []}) {
$data->{'artifacthubdata'}->{$1} = $2 if /^artifacthub:([^:]+):(.+)$/;
}
}
$data->{'notify'} = sub { BSNotify::notify('CONTAINER_PUBLISHED', { project => $projid , 'repo' => $repoid, 'buildid' => $publishid, 'container' => "$_[0]"}) };
if ($blobdir) {
$_->{'blobdir'} = $blobdir for values %containers;
Expand Down
13 changes: 8 additions & 5 deletions src/backend/bs_regpush
View file
Expand Up @@ -475,17 +475,20 @@ sub construct_container_tar {
push @tar, {'name' => $blobid, 'file' => $fd, 'mtime' => $mtime, 'offset' => 0, 'size' => (-s $fd)};
}
push @tar, {'name' => 'manifest.json', 'data' => $manifest, 'mtime' => $mtime, 'size' => length($manifest)};
return \@tar;
return (\@tar, $mtime);
}

sub open_tarfile {
my ($tarfile) = @_;

my ($tar, $tarfd, $containerinfo);
if ($tarfile =~ /\.containerinfo$/) {
if ($tarfile =~ /^artifacthub:(.+)/) {
$containerinfo = { 'type' => 'artifacthub', 'artifacthubdata' => $1 };
($tar) = BSContar::container_from_artifacthub($containerinfo->{'artifacthubdata'});
} elsif ($tarfile =~ /\.containerinfo$/) {
my $containerinfo_json = readstr($tarfile);
$containerinfo = JSON::XS::decode_json($containerinfo_json);
$tar = construct_container_tar($containerinfo);
($tar) = construct_container_tar($containerinfo);
} elsif ($tarfile =~ /\.helminfo$/) {
my $chart = $tarfile;
$chart =~ s/\.helminfo$/.tgz/;
Expand Down Expand Up @@ -683,8 +686,8 @@ if ($use_image_tags && @tarfiles > 1) {
$use_image_tags = 0;
}

# use oci types if we have a helm chart
$oci = 1 if grep {/\.helminfo$/} @tarfiles;
# use oci types if we have a helm chart or artifacthub data
$oci = 1 if grep {/\.helminfo$/ || /^artifacthub:/} @tarfiles;

my %digests_to_sign;
my @multimanifests;
Expand Down

0 comments on commit 0dbe62d

Please sign in to comment.