Skip to content

Commit

Permalink
Merge pull request #14224 from mlschroe/master
Browse files Browse the repository at this point in the history 
[backend] container publishing: remove more duplicated code
  • Loading branch information
@mlschroe
mlschroe committed Apr 25, 2023
2 parents b036a9e + 476e2ea commit a0fd68f
Show file tree
Hide file tree
Showing 4 changed files with 73 additions and 77 deletions.
44 changes: 41 additions & 3 deletions src/backend/BSContar.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ our $mt_oci_layer_gzip = 'application/vnd.oci.image.layer.v1.tar+gzip';
our $mt_oci_layer_zstd = 'application/vnd.oci.image.layer.v1.tar+zstd';
our $mt_helm_config = 'application/vnd.cncf.helm.config.v1+json';

sub blobid {
return 'sha256:'.Digest::SHA::sha256_hex($_[0]);
}

sub checksum_entry {
my ($ent, $ctx) = @_;
my $offset = 0;
Expand Down Expand Up @@ -259,6 +263,7 @@ sub get_config {
die("File $config_file not included in tar\n") unless $config_ent;
my $config_json = BSTar::extract($config_ent->{'file'}, $config_ent);
my $config = JSON::XS::decode_json($config_json);
$config_ent->{'blobid'} = blobid($config_json); # convenience
return ($config_ent, $config);
}

Expand Down Expand Up @@ -353,7 +358,7 @@ sub normalize_container {
my $ent = $newblobs{$blobid};
push @newtar, {'name' => $blobid, 'mtime' => $mtime, 'offset' => $ent->{'offset'}, 'size' => $ent->{'size'}, 'file' => $ent->{'file'}, 'blobid' => $blobid};
}
$newmanifest_ent->{'blobid'} = BSContar::blobid_entry($newmanifest_ent);
$newmanifest_ent->{'blobid'} = blobid_entry($newmanifest_ent);
push @newtar, $newmanifest_ent;

return (\@newtar, $mtime, $config, $newconfig, \@newlayercomp);
Expand Down Expand Up @@ -431,6 +436,16 @@ sub container_from_helm {
return ($tar, $mtime, \@layercomp);
}

sub create_config_data {
my ($config_ent, $oci) = @_;
my $config_data = {
'mediaType' => $config_ent->{'mimetype'} || ($oci ? $mt_oci_config : $mt_docker_config),
'size' => $config_ent->{'size'},
'digest' => $config_ent->{'blobid'} || blobid_entry($config_ent),
};
return $config_data;
}

sub create_layer_data {
my ($layer_ent, $oci, $comp, $newcomp, $lcomp) = @_;
$comp ||= detect_entry_compression($layer_ent);
Expand All @@ -447,9 +462,9 @@ sub create_layer_data {
undef $lcomp;
}
my $layer_data = {
'mediaType' => $layer_ent->{'mimetype'} || ($oci ? ($newcomp eq 'zstd' ? $BSContar::mt_oci_layer_zstd : $BSContar::mt_oci_layer_gzip) : $BSContar::mt_docker_layer_gzip),
'mediaType' => $layer_ent->{'mimetype'} || ($oci ? ($newcomp eq 'zstd' ? $mt_oci_layer_zstd : $mt_oci_layer_gzip) : $mt_docker_layer_gzip),
'size' => $layer_ent->{'size'},
'digest' => $layer_ent->{'blobid'} || BSContar::blobid_entry($layer_ent),
'digest' => $layer_ent->{'blobid'} || blobid_entry($layer_ent),
};
if ($newcomp eq 'zstd' && $lcomp && $lcomp =~ /^zstd:chunked/) {
my @c = split(',', $lcomp);
Expand All @@ -459,4 +474,27 @@ sub create_layer_data {
return ($layer_ent, $layer_data);
}

sub create_dist_manifest_data {
my ($config_data, $layer_data, $oci) = @_;
my $mediaType = $oci ? $mt_oci_manifest : $mt_docker_manifest;
my $mani = {
'schemaVersion' => 2,
'mediaType' => $mediaType,
'config' => $config_data,
'layers' => $layer_data,
};
return $mani;
}

sub create_dist_manifest_list_data {
my ($multimanifest_data, $oci) = @_;
my $mediaType = $oci ? $mt_oci_index : $mt_docker_manifestlist;
my $manilist = {
'schemaVersion' => 2,
'mediaType' => $mediaType,
'manifests' => $multimanifest_data,
};
return $manilist;
}

1;
29 changes: 7 additions & 22 deletions src/backend/BSPublisher/Container.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,11 +343,7 @@ sub create_container_dist_info {
$platforms->{$platformstr} = 1;
}

my $config_data = {
'mediaType' => $config_ent->{'mimetype'} || ($oci ? $BSContar::mt_oci_config : $BSContar::mt_docker_config),
'size' => $config_ent->{'size'},
'digest' => BSContar::blobid_entry($config_ent),
};
my $config_data = BSContar::create_config_data($config_ent, $oci);
my @layer_data;
die("container has no layers\n") unless @{$manifest->{'Layers'} || []};
my @layer_comp = @{$layer_compression || []};
Expand All @@ -362,18 +358,12 @@ sub create_container_dist_info {
($layer_ent, $layer_data) = BSContar::create_layer_data($layer_ent, $oci, $comp, undef, $lcomp);
push @layer_data, $layer_data;
}
my $mediaType = $oci ? $BSContar::mt_oci_manifest : $BSContar::mt_docker_manifest;
my $mani = {
'schemaVersion' => 2,
'mediaType' => $mediaType,
'config' => $config_data,
'layers' => \@layer_data,
};
my $mani = BSContar::create_dist_manifest_data($config_data, \@layer_data, $oci);
my $mani_json = BSContar::create_dist_manifest($mani);
my $info = {
'mediaType' => $mediaType,
'mediaType' => $mani->{'mediaType'},
'size' => length($mani_json),
'digest' => 'sha256:'.Digest::SHA::sha256_hex($mani_json),
'digest' => BSContar::blobid($mani_json),
'platform' => {'architecture' => $goarch, 'os' => $goos},
};
$info->{'platform'}->{'variant'} = $govariant if $govariant;
Expand All @@ -382,17 +372,12 @@ sub create_container_dist_info {

sub create_container_index_info {
my ($infos, $oci) = @_;
my $mediaType = $oci ? $BSContar::mt_oci_index : $BSContar::mt_docker_manifestlist;
my $mani = {
'schemaVersion' => 2,
'mediaType' => $mediaType,
'manifests' => $infos || [],
};
my $mani = BSContar::create_dist_manifest_list_data($infos || [], $oci);
my $mani_json = BSContar::create_dist_manifest_list($mani);
my $info = {
'mediaType' => $mediaType,
'mediaType' => $mani->{'mediaType'},
'size' => length($mani_json),
'digest' => 'sha256:'.Digest::SHA::sha256_hex($mani_json),
'digest' => BSContar::blobid($mani_json),
};
return $info;
}
Expand Down
34 changes: 10 additions & 24 deletions src/backend/BSPublisher/Registry.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ sub push_blob {

sub push_blob_content {
my ($repodir, $content) = @_;
my $blob_id = 'sha256:'.Digest::SHA::sha256_hex($content);
my $blob_id = BSContar::blobid($content);
my $dir = "$repodir/:blobs";
return $blob_id if -e "$dir/$blob_id";
mkdir_p($dir) unless -d $dir;
Expand All @@ -159,7 +159,7 @@ sub push_blob_content {

sub push_manifest {
my ($repodir, $mani_json) = @_;
my $mani_id = 'sha256:'.Digest::SHA::sha256_hex($mani_json);
my $mani_id = BSContar::blobid($mani_json);
my $dir = "$repodir/:manifests";
return $mani_id if -e "$dir/$mani_id";
mkdir_p($dir) unless -d $dir;
Expand Down Expand Up @@ -674,13 +674,10 @@ sub push_containers {
$multiplatforms{$platformstr} = 1;

# put config blob into repo
my $config_blobid = push_blob($repodir, $containerinfo, $config_ent);
my $config_data = BSContar::create_config_data($config_ent, $oci);
my $config_blobid = $config_ent->{'blobid'} = $config_data->{'digest'};
push_blob($repodir, $containerinfo, $config_ent);
$knownblobs{$config_blobid} = 1;
my $config_data = {
'mediaType' => $config_ent->{'mimetype'} || ($oci ? $BSContar::mt_oci_config : $BSContar::mt_docker_config),
'size' => $config_ent->{'size'},
'digest' => $config_blobid,
};

# put layer blobs into repo
my %layer_datas;
Expand All @@ -702,27 +699,21 @@ sub push_containers {
push @layer_data, $layer_data;
$layer_datas{$layer_file} = $layer_data;

$layer_ent->{'blobid'} = $layer_data->{'digest'};
$knownblobs{$layer_ent->{'blobid'}} = 1;
my $layer_blobid = $layer_ent->{'blobid'} = $layer_data->{'digest'};
push_blob($repodir, $containerinfo, $layer_ent);
$knownblobs{$layer_blobid} = 1;
}
close $tarfd if $tarfd;

# put manifest into repo
my $mediaType = $oci ? $BSContar::mt_oci_manifest : $BSContar::mt_docker_manifest;
my $mani = {
'schemaVersion' => 2,
'mediaType' => $mediaType,
'config' => $config_data,
'layers' => \@layer_data,
};
my $mani = BSContar::create_dist_manifest_data($config_data, \@layer_data, $oci);
my $mani_json = BSContar::create_dist_manifest($mani);
my $mani_id = push_manifest($repodir, $mani_json);
$knownmanifests{$mani_id} = 1;
$digests_to_cosign{$mani_id} = [ $oci, $containerinfo ];

my $multimani = {
'mediaType' => $mediaType,
'mediaType' => $mani->{'mediaType'},
'size' => length($mani_json),
'digest' => $mani_id,
'platform' => {'architecture' => $goarch, 'os' => $goos},
Expand Down Expand Up @@ -767,12 +758,7 @@ sub push_containers {
my ($mani_id, $mani_size);
if ($multiarchtag) {
# create fat manifest
my $mediaType = $oci ? $BSContar::mt_oci_index : $BSContar::mt_docker_manifestlist;
my $mani = {
'schemaVersion' => 2,
'mediaType' => $mediaType,
'manifests' => \@multimanifests,
};
my $mani = BSContar::create_dist_manifest_list_data(\@multimanifests, $oci);
my $mani_json = BSContar::create_dist_manifest_list($mani);
$mani_id = push_manifest($repodir, $mani_json, \%knownmanifests);
$mani_size = length($mani_json);
Expand Down
43 changes: 15 additions & 28 deletions src/backend/bs_regpush
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ sub manifest_exists {
my ($manifest, $tag, $content_type) = @_;

$content_type ||= $BSContar::mt_docker_manifest;
my $maniid = 'sha256:'.Digest::SHA::sha256_hex($manifest);
my $maniid = BSContar::blobid($manifest);
$tag = $maniid unless defined $tag;
my $replyheaders;
my $param = {
Expand All @@ -199,7 +199,7 @@ sub manifest_exists {
eval { $maniret = BSRPC::rpc($param) };
if ($maniret) {
my $maniretid = $replyheaders->{'docker-content-digest'};
$maniretid ||= 'sha256:'.Digest::SHA::sha256_hex($maniret);
$maniretid ||= BSContar::blobid($maniret);
return 1 if $maniretid eq $maniid;
}
return 0;
Expand All @@ -208,7 +208,7 @@ sub manifest_exists {
sub manifest_upload {
my ($manifest, $tag, $content_type, $quiet) = @_;

my $maniid = 'sha256:'.Digest::SHA::sha256_hex($manifest);
my $maniid = BSContar::blobid($manifest);
return $maniid if manifest_exists($manifest, $tag, $content_type);
$content_type ||= $BSContar::mt_docker_manifest;
my $fat = '';
Expand Down Expand Up @@ -240,7 +240,7 @@ sub manifest_upload {

sub manifest_append {
my ($manifest, $tag) = @_;
my $maniid = 'sha256:'.Digest::SHA::sha256_hex($manifest);
my $maniid = BSContar::blobid($manifest);
my $str = "$maniid ".length($manifest).(defined($tag) ? " $tag" : '')."\n";
if ($digestfile ne '-') {
BSUtil::appendstr($digestfile, $str);
Expand All @@ -264,7 +264,7 @@ sub manifest_upload_tags {

sub cosign_upload {
my ($tag, $config, @layers) = @_;
my $config_blobid = blob_upload_content('sha256:'.Digest::SHA::sha256_hex($config), $config);
my $config_blobid = blob_upload_content(BSContar::blobid($config), $config);
my $config_data = {
'mediaType' => $BSContar::mt_oci_config,
'size' => length($config),
Expand All @@ -279,7 +279,7 @@ sub cosign_upload {
};
while (@layers >= 2) {
my ($payload_layer, $payload) = splice(@layers, 0, 2);
my $payload_blobid = blob_upload_content('sha256:'.Digest::SHA::sha256_hex($payload), $payload);
my $payload_blobid = blob_upload_content(BSContar::blobid($payload), $payload);
die unless $payload_blobid eq $payload_layer->{'digest'};
push @{$mani->{'layers'}}, $payload_layer;
}
Expand Down Expand Up @@ -763,13 +763,9 @@ for my $tarfile (@tarfiles) {


# process config
my $config_blobid = BSContar::blobid_entry($config_ent);
# create layer data
my $config_data = {
'mediaType' => $config_ent->{'mimetype'} || ($oci ? $BSContar::mt_oci_config : $BSContar::mt_docker_config),
'size' => $config_ent->{'size'},
'digest' => $config_blobid,
};
my $config_data = BSContar::create_config_data($config_ent, $oci);
my $config_blobid = $config_data->{'digest'};

# upload to server
blob_upload($config_blobid, $config_ent);

Expand Down Expand Up @@ -798,15 +794,10 @@ for my $tarfile (@tarfiles) {
}
close $tarfd if $tarfd;

my $mediaType = $oci ? $BSContar::mt_oci_manifest : $BSContar::mt_docker_manifest;
my $mani = {
'schemaVersion' => 2,
'mediaType' => $mediaType,
'config' => $config_data,
'layers' => \@layer_data,
};
my $mani = BSContar::create_dist_manifest_data($config_data, \@layer_data, $oci);
my $mediaType = $mani->{'mediaType'};
my $mani_json = BSContar::create_dist_manifest($mani);
my $mani_id = 'sha256:'.Digest::SHA::sha256_hex($mani_json);
my $mani_id = BSContar::blobid($mani_json);
$digests_to_sign{$mani_id} = [ $provenance, $spdx_json, $cyclonedx_json ];

if ($multiarch) {
Expand Down Expand Up @@ -847,14 +838,10 @@ my $info = {
};

if ($multiarch) {
my $mediaType = $oci ? $BSContar::mt_oci_index : $BSContar::mt_docker_manifestlist;
my $mani = {
'schemaVersion' => 2,
'mediaType' => $mediaType,
'manifests' => \@multimanifests,
};
my $mani = BSContar::create_dist_manifest_list_data(\@multimanifests, $oci);
my $mediaType = $mani->{'mediaType'};
my $mani_json = BSContar::create_dist_manifest_list($mani);
my $mani_id = 'sha256:'.Digest::SHA::sha256_hex($mani_json);
my $mani_id = BSContar::blobid($mani_json);
$digests_to_sign{$mani_id} = [];
manifest_upload_tags($mani_json, \@tags, $mediaType);
$info->{'distmanifesttype'} = 'list';
Expand Down

0 comments on commit a0fd68f

Please sign in to comment.