diff --git a/src/backend/BSContar.pm b/src/backend/BSContar.pm index 7086604be9c..06e66054c1f 100644 --- a/src/backend/BSContar.pm +++ b/src/backend/BSContar.pm @@ -26,6 +26,7 @@ use JSON::XS (); use Digest::SHA (); use Digest::MD5 (); use Compress::Zlib (); +use Scalar::Util; use POSIX; use BSUtil; @@ -44,6 +45,8 @@ our $mt_oci_config = 'application/vnd.oci.image.config.v1+json'; our $mt_oci_layer_gzip = 'application/vnd.oci.image.layer.v1.tar+gzip'; our $mt_oci_layer_zstd = 'application/vnd.oci.image.layer.v1.tar+zstd'; our $mt_helm_config = 'application/vnd.cncf.helm.config.v1+json'; +our $mt_artifacthub_config = 'application/vnd.cncf.artifacthub.config.v1+yaml'; +our $mt_artifacthub_layer = 'application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml'; sub blobid { return 'sha256:'.Digest::SHA::sha256_hex($_[0]); @@ -270,6 +273,7 @@ sub get_config { die("File $config_file not included in tar\n") unless $config_ent; my $config_json = BSTar::extract($config_ent->{'file'}, $config_ent); $config_ent->{'blobid'} ||= blobid($config_json); # convenience + return ($config_ent, {}) if $config_json eq ''; # workaround for artifacthub my $config = JSON::XS::decode_json($config_json); return ($config_ent, $config); } @@ -442,6 +446,57 @@ sub container_from_helm { return ($tar, $mtime, \@layercomp); } +sub unparse_yaml_string { + my ($d) = @_; + return "''" unless length $d; + return "\"$d\"" if Scalar::Util::looks_like_number($d); + if ($d =~ /[\x00-\x1f\x7f-\x9f\']/) { + $d =~ s/\\/\\\\/g; + $d =~ s/\"/\\\"/g; + $d =~ s/([\x00-\x1f\x7f-\x9f])/'\x'.sprintf("%X",ord($1))/ge; + return "\"$d\""; + } elsif ($d =~ /^[\!\&*{}[]|>@`"'#%, ]/s) { + return "'$d'"; + } elsif ($d =~ /: / || $d =~ / #/ || $d =~ /[: \t]\z/) { + return "'$d'"; + } elsif ($d eq '~' || $d eq 'null' || $d eq 'true' || $d eq 'false' && $d =~ /^(?:---|\.\.\.)/s) { + return "'$d'"; + } elsif ($d =~ /^[-?:](?:\s|\z)/s) { + return "'$d'"; + } else { + return $d; + } +} + +sub create_artifacthub_yaml { + my ($artifacthubdata) = @_; + my ($repoid, $name, $email) = split(':', $artifacthubdata, 3); + my $yaml = ''; + $yaml .= "repositoryID: ".unparse_yaml_string($repoid)."\n" if $repoid; + my $owners = ''; + $owners .= " name: ".unparse_yaml_string($name)."\n" if $name; + $owners .= " email: ".unparse_yaml_string($email)."\n" if $email; + $owners =~ s/^ / -/; + $yaml .= "owners:\n$owners" if $owners; + return $yaml; +} + +sub container_from_artifacthub { + my ($artifacthubdata, $mtime) = @_; + my $artifacthub_yaml = create_artifacthub_yaml($artifacthubdata); + my $config_ent = { 'name' => 'config.yaml', 'mtime' => $mtime, 'data' => '', 'size' => 0, 'mimetype' => $mt_artifacthub_config }; + my $layer_ent = { 'name' => 'artifacthub-repo.yml', 'mtime' => $mtime, 'data' => $artifacthub_yaml, 'size' => length($artifacthub_yaml), 'mimetype' => $mt_artifacthub_layer }; + $layer_ent->{'annotations'}->{'org.opencontainers.image.title'} = 'artifacthub-repo.yml'; + my $manifest = { + 'Layers' => [ 'artifacthub-repo.yml' ], + 'Config' => 'config.yaml', + 'RepoTags' => [ 'artifacthub.io' ], + }; + my $manifest_ent = create_manifest_entry($manifest, $mtime); + my $tar = [ $manifest_ent, $config_ent, $layer_ent ]; + return ($tar, $mtime, [ '' ]); +} + sub create_config_data { my ($config_ent, $oci) = @_; my $config_data = { @@ -482,7 +537,7 @@ sub create_layer_data { 'size' => 0 + $layer_ent->{'size'}, 'digest' => $layer_ent->{'blobid'} || blobid_entry($layer_ent), }; - $layer_data->{'annotations'} = { %$annotations } if $annotations; + $layer_data->{'annotations'} = { %{$layer_ent->{'annotations'} || {}}, %{$annotations || {}} } if $layer_ent->{'annotations'} || $annotations; if ($comp eq 'zstd' && $lcomp && $lcomp =~ /^zstd:chunked/) { my @c = split(',', $lcomp); $layer_data->{'annotations'}->{'io.github.containers.zstd-chunked.manifest-position'} = $c[1] if $c[1]; diff --git a/src/backend/BSPublisher/Container.pm b/src/backend/BSPublisher/Container.pm index 6daada852b3..b3911025291 100644 --- a/src/backend/BSPublisher/Container.pm +++ b/src/backend/BSPublisher/Container.pm @@ -199,7 +199,7 @@ sub upload_all_containers { for my $p (sort keys %$containers) { my $containerinfo = $containers->{$p}; my $arch = $containerinfo->{'arch'}; - my $goarch = $containerinfo->{'goarch'} || (($containerinfo->{'type'} || '') eq 'helm' ? 'any' : $arch); + my $goarch = $containerinfo->{'goarch'} || (($containerinfo->{'type'} || '') eq 'helm' || ($containerinfo->{'type'} || '') eq 'artifacthub' ? 'any' : $arch); $goarch .= ":$containerinfo->{'govariant'}" if $containerinfo->{'govariant'}; $goarch .= "_$containerinfo->{'goos'}" if $containerinfo->{'goos'} && $containerinfo->{'goos'} ne 'linux'; my @tags = $mapper->($registry, $containerinfo, $projid, $repoid, $arch); @@ -308,11 +308,12 @@ sub reconstruct_container { BSTar::writetarfile($dst, $dstfinal, $tar, 'mtime' => $mtime) if $tar; } -sub create_container_dist_info { - my ($containerinfo, $oci, $platforms) = @_; - my $file = $containerinfo->{'publishfile'}; +sub open_container_tar { + my ($containerinfo, $file) = @_; my ($tar, $mtime, $layer_compression); - if (!defined($file)) { + if (($containerinfo->{'type'} || '') eq 'artifacthub') { + ($tar, $mtime, $layer_compression) = BSContar::container_from_artifacthub($containerinfo->{'artifacthubdata'}); + } elsif (!defined($file)) { ($tar, $mtime, $layer_compression) = BSPublisher::Containerinfo::construct_container_tar($containerinfo, 1); } elsif (($containerinfo->{'type'} || '') eq 'helm') { ($tar, $mtime, $layer_compression) = BSContar::container_from_helm($file, $containerinfo->{'config_json'}, $containerinfo->{'tags'}); @@ -330,6 +331,12 @@ sub create_container_dist_info { $_->{'file'} = $tarfd for @$tar; } die("incomplete containerinfo\n") unless $tar; + return ($tar, $mtime, $layer_compression); +} + +sub create_container_dist_info { + my ($containerinfo, $oci, $platforms) = @_; + my ($tar, $mtime, $layer_compression) = open_container_tar($containerinfo, $containerinfo->{'publishfile'}); my %tar = map {$_->{'name'} => $_} @$tar; my ($manifest_ent, $manifest) = BSContar::get_manifest(\%tar); my ($config_ent, $config) = BSContar::get_config(\%tar, $manifest); @@ -529,9 +536,10 @@ sub upload_to_registry { my $multiarch = 0; # XXX: use $data->{'multiarch'} $multiarch = 1 if @$containerinfos > 1; $multiarch = 0 if @$containerinfos == 1 && ($containerinfos->[0]->{'type'} || '') eq 'helm'; + $multiarch = 0 if @$containerinfos == 1 && ($containerinfos->[0]->{'type'} || '') eq 'artifacthub'; my $oci; for my $containerinfo (@$containerinfos) { - $oci = 1 if ($containerinfo->{'type'} || '') eq 'helm'; + $oci = 1 if ($containerinfo->{'type'} || '') eq 'helm' || ($containerinfo->{'type'} || '') eq 'artifacthub'; $oci = 1 if grep {$_ && $_ ne 'gzip'} @{$containerinfo->{'layer_compression'} || []}; } @@ -569,6 +577,10 @@ sub upload_to_registry { my $file = $containerinfo->{'publishfile'}; my $wrote_containerinfo; if (!defined($file)) { + if (($containerinfo->{'type'} || '') eq 'artifacthub') { + push @uploadfiles, "artifacthub:$containerinfo->{'artifacthubdata'}"; + next; + } # tar file needs to be constructed from blobs $blobdir = $containerinfo->{'blobdir'}; die("need a blobdir for containerinfo uploads\n") unless $blobdir; @@ -832,6 +844,13 @@ sub do_local_uploads { push @{$todo{$tag}}, $containerinfo; } } + my $gun = $registry->{'notary_gunprefix'} || $registry->{'server'}; + $gun =~ s/^https?:\/\///; + $gun = '' if $gun eq 'local:'; + if (($data->{'artifacthubdata'} || {})->{"$gun/$repository"} && !$uptags->{'artifacthub.io'}) { + my $containerinfo = { 'type' => 'artifacthub', 'artifacthubdata' => $data->{'artifacthubdata'}->{"$gun/$repository"} }; + push @{$todo{'artifacthub.io'}}, $containerinfo; + } eval { BSPublisher::Registry::push_containers($registry, $projid, $repoid, $repository, \%todo, $data); }; @@ -877,6 +896,14 @@ sub do_remote_uploads { my $digests = upload_to_registry($registry, $projid, $repoid, $repository, \@containerinfos, \@tags, $data, $repostate); $containerdigests .= $digests; } + my $gun = $registry->{'notary_gunprefix'} || $registry->{'server'}; + $gun =~ s/^https?:\/\///; + $gun = '' if $gun eq 'local:'; + if (($data->{'artifacthubdata'} || {})->{"$gun/$repository"} && !$uptags->{'artifacthub.io'}) { + my $containerinfo = { 'type' => 'artifacthub', 'artifacthubdata' => $data->{'artifacthubdata'}->{"$gun/$repository"} }; + my $digests = upload_to_registry($registry, $projid, $repoid, $repository, [ $containerinfo ], [ 'artifacthub.io' ], $data, $repostate); + $containerdigests .= $digests; + } # all is pushed, now clean the rest add_notary_upload($notary_uploads, $registry, $repository, $containerdigests); delete_obsolete_tags_from_registry($registry, $repository, $containerdigests, $repostate); diff --git a/src/backend/BSPublisher/Registry.pm b/src/backend/BSPublisher/Registry.pm index 7402b471cca..079f494f91c 100644 --- a/src/backend/BSPublisher/Registry.pm +++ b/src/backend/BSPublisher/Registry.pm @@ -548,6 +548,28 @@ sub create_manifestinfo { push_manifestinfo($repodir, $imginfo->{'distmanifest'}, JSON::XS->new->utf8->canonical->encode($imginfo)); } +sub open_container_tar { + my ($containerinfo, $file) = @_; + my ($tar, $mtime, $layer_compression); + if (($containerinfo->{'type'} || '') eq 'artifacthub') { + ($tar, $mtime, $layer_compression) = BSContar::container_from_artifacthub($containerinfo->{'artifacthubdata'}); + } elsif (!defined($file)) { + ($tar, $mtime, $layer_compression) = BSPublisher::Containerinfo::construct_container_tar($containerinfo, 1); + # set blobfile in entries so we can create a link in push_blob + for (@$tar) { + $_->{'blobfile'} = "$containerinfo->{'blobdir'}/_blob.$_->{'blobid'}" if $_->{'blobid'}; + } + } elsif (($containerinfo->{'type'} || '') eq 'helm') { + ($tar, $mtime, $layer_compression) = BSContar::container_from_helm($file, $containerinfo->{'config_json'}, $containerinfo->{'tags'}); + } else { + my $tarfd; + open($tarfd, '<', $file) || die("$file: $!\n"); + ($tar, $mtime, undef, undef, $layer_compression) = BSContar::normalize_container($tarfd, 1); + } + die("incomplete containerinfo\n") unless $tar; + return ($tar, $mtime, $layer_compression); +} + sub push_containers { my ($registry, $projid, $repoid, $repo, $tags, $data) = @_; @@ -594,6 +616,7 @@ sub push_containers { my $multiarch = $data->{'multiarch'}; $multiarch = 1 if @$containerinfos > 1; $multiarch = 0 if @$containerinfos == 1 && ($containerinfos->[0]->{'type'} || '') eq 'helm'; + $multiarch = 0 if @$containerinfos == 1 && ($containerinfos->[0]->{'type'} || '') eq 'artifacthub'; die("must use multiarch if multiple containers are to be pushed\n") if @$containerinfos > 1 && !$multiarch; my %multiplatforms; my @multimanifests; @@ -601,7 +624,7 @@ sub push_containers { my $oci; # use oci types if we have a helm chart or we use a nonstandard compression for my $containerinfo (@$containerinfos) { - $oci = 1 if ($containerinfo->{'type'} || '') eq 'helm'; + $oci = 1 if ($containerinfo->{'type'} || '') eq 'helm' || ($containerinfo->{'type'} || '') eq 'artifacthub'; $oci = 1 if grep {$_ && $_ ne 'gzip'} @{$containerinfo->{'layer_compression'} || []}; } for my $containerinfo (@$containerinfos) { @@ -620,22 +643,7 @@ sub push_containers { next; } - my ($tar, $mtime, $layer_compression); - my $tarfd; - if ($containerinfo->{'uploadfile'}) { - open($tarfd, '<', $containerinfo->{'uploadfile'}) || die("$containerinfo->{'uploadfile'}: $!\n"); - if (($containerinfo->{'type'} || '') eq 'helm') { - ($tar, $mtime, $layer_compression) = BSContar::container_from_helm($containerinfo->{'uploadfile'}, $containerinfo->{'config_json'}, $containerinfo->{'tags'}); - } else { - ($tar, $mtime, undef, undef, $layer_compression) = BSContar::normalize_container($tarfd, 1); - } - } else { - ($tar, $mtime, $layer_compression) = BSPublisher::Containerinfo::construct_container_tar($containerinfo, 1); - # set blobfile in entries so we can create a link in push_blob - for (@$tar) { - $_->{'blobfile'} = "$containerinfo->{'blobdir'}/_blob.$_->{'blobid'}" if $_->{'blobid'}; - } - } + my ($tar, $mtime, $layer_compression) = open_container_tar($containerinfo, $containerinfo->{'uploadfile'}); my %tar = map {$_->{'name'} => $_} @$tar; my ($manifest_ent, $manifest) = BSContar::get_manifest(\%tar); @@ -657,7 +665,6 @@ sub push_containers { $platformstr .= " variant:$govariant" if $govariant; if ($multiplatforms{$platformstr}) { print "ignoring $containerinfo->{'file'}, already have $platformstr\n"; - close $tarfd if $tarfd; next; } $multiplatforms{$platformstr} = 1; @@ -690,7 +697,6 @@ sub push_containers { push_blob($repodir, $layer_ent); $knownblobs{$layer_blobid} = 1; } - close $tarfd if $tarfd; # put manifest into repo my $mani = BSContar::create_dist_manifest_data($config_data, \@layer_data, $oci); diff --git a/src/backend/bs_publish b/src/backend/bs_publish index 6501585f362..174ad134447 100755 --- a/src/backend/bs_publish +++ b/src/backend/bs_publish @@ -2857,6 +2857,11 @@ sub publish { 'publishid' => $publishid, 'multiarch' => $multicontainer, }; + if ($config->{'publishflags:artifacthub'}) { + for (@{$config->{'publishflags'} || []}) { + $data->{'artifacthubdata'}->{$1} = $2 if /^artifacthub:([^:]+):(.+)$/; + } + } $data->{'notify'} = sub { BSNotify::notify('CONTAINER_PUBLISHED', { project => $projid , 'repo' => $repoid, 'buildid' => $publishid, 'container' => "$_[0]"}) }; if ($blobdir) { $_->{'blobdir'} = $blobdir for values %containers; diff --git a/src/backend/bs_regpush b/src/backend/bs_regpush index 66daec7248d..c8b3fae5fbd 100755 --- a/src/backend/bs_regpush +++ b/src/backend/bs_regpush @@ -475,17 +475,20 @@ sub construct_container_tar { push @tar, {'name' => $blobid, 'file' => $fd, 'mtime' => $mtime, 'offset' => 0, 'size' => (-s $fd)}; } push @tar, {'name' => 'manifest.json', 'data' => $manifest, 'mtime' => $mtime, 'size' => length($manifest)}; - return \@tar; + return (\@tar, $mtime); } sub open_tarfile { my ($tarfile) = @_; my ($tar, $tarfd, $containerinfo); - if ($tarfile =~ /\.containerinfo$/) { + if ($tarfile =~ /^artifacthub:(.+)/) { + $containerinfo = { 'type' => 'artifacthub', 'artifacthubdata' => $1 }; + ($tar) = BSContar::container_from_artifacthub($containerinfo->{'artifacthubdata'}); + } elsif ($tarfile =~ /\.containerinfo$/) { my $containerinfo_json = readstr($tarfile); $containerinfo = JSON::XS::decode_json($containerinfo_json); - $tar = construct_container_tar($containerinfo); + ($tar) = construct_container_tar($containerinfo); } elsif ($tarfile =~ /\.helminfo$/) { my $chart = $tarfile; $chart =~ s/\.helminfo$/.tgz/; @@ -683,8 +686,8 @@ if ($use_image_tags && @tarfiles > 1) { $use_image_tags = 0; } -# use oci types if we have a helm chart -$oci = 1 if grep {/\.helminfo$/} @tarfiles; +# use oci types if we have a helm chart or artifacthub data +$oci = 1 if grep {/\.helminfo$/ || /^artifacthub:/} @tarfiles; my %digests_to_sign; my @multimanifests;