From 0a8fe8b5d7c3eb5649283212465a298f7c84057d Mon Sep 17 00:00:00 2001 From: Ina Panova Date: Fri, 8 Oct 2021 16:13:00 +0200 Subject: [PATCH] Fixed blob content_type headers and added blob/manifest validation. closes #9571 closes #8303 --- CHANGES/8303.bugfix | 1 + CHANGES/9571.bugfix | 1 + pulp_container/app/redirects.py | 4 +- pulp_container/app/registry.py | 13 ++++-- pulp_container/app/registry_api.py | 74 ++++++++++++++++++++++++++++-- pulp_container/constants.py | 1 + 6 files changed, 83 insertions(+), 11 deletions(-) create mode 100644 CHANGES/8303.bugfix create mode 100644 CHANGES/9571.bugfix diff --git a/CHANGES/8303.bugfix b/CHANGES/8303.bugfix new file mode 100644 index 000000000..0ae9594da --- /dev/null +++ b/CHANGES/8303.bugfix @@ -0,0 +1 @@ +Added validation for the supported manifests and blobs media_types in the push operation. diff --git a/CHANGES/9571.bugfix b/CHANGES/9571.bugfix new file mode 100644 index 000000000..3b42eda60 --- /dev/null +++ b/CHANGES/9571.bugfix @@ -0,0 +1 @@ +Send proper blob content_type header when the blob is served. diff --git a/pulp_container/app/redirects.py b/pulp_container/app/redirects.py index 3e3c67f60..48af2d333 100644 --- a/pulp_container/app/redirects.py +++ b/pulp_container/app/redirects.py @@ -6,7 +6,7 @@ from django.shortcuts import redirect from pulp_container.app.utils import get_accepted_media_types -from pulp_container.constants import MEDIA_TYPE +from pulp_container.constants import BLOB_CONTENT_TYPE, MEDIA_TYPE class CommonRedirects: @@ -101,7 +101,7 @@ def issue_blob_redirect(self, blob): except ObjectDoesNotExist: return self.redirect_to_content_app("blobs", blob.digest) - return self.redirect_to_object_storage(artifact, blob.media_type) + return self.redirect_to_object_storage(artifact, BLOB_CONTENT_TYPE) def redirect_to_object_storage(self, artifact, return_media_type): """ diff --git a/pulp_container/app/registry.py b/pulp_container/app/registry.py index abdf6156f..d7e9a7a4d 100644 --- a/pulp_container/app/registry.py +++ b/pulp_container/app/registry.py @@ -10,13 +10,14 @@ from pulpcore.plugin.content import Handler, PathNotResolved from pulpcore.plugin.models import ContentArtifact -from pulp_container.app.models import ContainerDistribution, Tag +from pulp_container.app.models import ContainerDistribution, Tag, Blob from pulp_container.app.schema_convert import Schema2toSchema1ConverterWrapper from pulp_container.app.utils import get_accepted_media_types -from pulp_container.constants import EMPTY_BLOB, MEDIA_TYPE +from pulp_container.constants import BLOB_CONTENT_TYPE, EMPTY_BLOB, MEDIA_TYPE log = logging.getLogger(__name__) + v2_headers = MultiDict() v2_headers["Docker-Distribution-API-Version"] = "registry/2.0" @@ -220,8 +221,12 @@ async def get_by_digest(self, request): relative_path=digest, ) ca_content = await sync_to_async(ca.content.cast)() + if isinstance(ca_content, Blob): + media_type = BLOB_CONTENT_TYPE + else: + media_type = ca_content.media_type headers = { - "Content-Type": ca_content.media_type, + "Content-Type": media_type, "Docker-Content-Digest": ca_content.digest, } except ObjectDoesNotExist: @@ -244,7 +249,7 @@ async def _empty_blob(): body = bytearray(empty_tar) response_headers = { "Docker-Content-Digest": EMPTY_BLOB, - "Content-Type": MEDIA_TYPE.REGULAR_BLOB, + "Content-Type": BLOB_CONTENT_TYPE, "Docker-Distribution-API-Version": "registry/2.0", } return web.Response(body=body, headers=response_headers) diff --git a/pulp_container/app/registry_api.py b/pulp_container/app/registry_api.py index b3125fe52..dca2a781a 100644 --- a/pulp_container/app/registry_api.py +++ b/pulp_container/app/registry_api.py @@ -115,6 +115,24 @@ def __init__(self, digest): ) +class BlobInvalid(ParseError): + """Exception to render a 400 with the code 'BLOB_UNKNOWN'""" + + def __init__(self, digest): + """Initialize the exception with the blob digest.""" + super().__init__( + detail={ + "errors": [ + { + "code": "BLOB_UNKNOWN", + "message": "blob unknown to registry", + "detail": {"digest": digest}, + } + ] + } + ) + + class ManifestNotFound(NotFound): """Exception to render a 404 with the code 'MANIFEST_UNKNOWN'""" @@ -133,6 +151,24 @@ def __init__(self, reference): ) +class ManifestInvalid(ParseError): + """Exception to render a 400 with the code 'MANIFEST_INVALID'""" + + def __init__(self, digest): + """Initialize the exception with the manifest digest.""" + super().__init__( + detail={ + "errors": [ + { + "code": "MANIFEST_INVALID", + "message": "manifest invalid", + "detail": {"digest": digest}, + } + ] + } + ) + + class ContentRenderer(BaseRenderer): """ Rendered class for rendering Manifest and Blob responses. @@ -814,14 +850,34 @@ def put(self, request, path, pk=None): # iterate over all the layers and create chunk = request.META["wsgi.input"] artifact = self.receive_artifact(chunk) + manifest_digest = "sha256:{id}".format(id=artifact.sha256) with storage.open(artifact.file.name) as artifact_file: raw_data = artifact_file.read() content_data = json.loads(raw_data) + # oci format might not contain mediaType in the manifest.json, docker should + # hence need to check request content type + if request.content_type not in ( + models.MEDIA_TYPE.MANIFEST_V2, + models.MEDIA_TYPE.MANIFEST_OCI, + ): + # we suport only v2 docker/oci schema upload + raise ManifestInvalid(digest=manifest_digest) + # both docker/oci format should contain config, digest, mediaType, size config_layer = content_data.get("config") - config_blob = models.Blob.objects.get(digest=config_layer.get("digest")) + try: + config_digest = config_layer.get("digest") + config_blob = models.Blob.objects.get(digest=config_digest) + except models.Blob.DoesNotExist: + raise BlobNotFound(digest=config_digest) + config_media_type = config_layer.get("mediaType") + if config_media_type not in ( + models.MEDIA_TYPE.CONFIG_BLOB, + models.MEDIA_TYPE.CONFIG_BLOB_OCI, + ): + raise BlobInvalid(digest=config_blob.digest) manifest = models.Manifest( - digest="sha256:{id}".format(id=artifact.sha256), + digest=manifest_digest, schema_version=2, media_type=request.content_type, config_blob=config_blob, @@ -836,13 +892,21 @@ def put(self, request, path, pk=None): ca.save() except IntegrityError: pass + # both docker/oci format should contain layers, digest, media_type, size layers = content_data.get("layers") - blobs = [] + blobs = {} for layer in layers: - blobs.append(layer.get("digest")) - blobs_qs = models.Blob.objects.filter(digest__in=blobs) + blobs[layer.get("digest")] = layer.get("mediaType") + blobs_qs = models.Blob.objects.filter(digest__in=blobs.keys()) thru = [] for blob in blobs_qs: + # ensure there are no foreign layers + blob_media_type = blobs[blob.digest] + if blob_media_type not in ( + models.MEDIA_TYPE.REGULAR_BLOB, + models.MEDIA_TYPE.REGULAR_BLOB_OCI, + ): + raise BlobInvalid(digest=blob.digest) thru.append(models.BlobManifest(manifest=manifest, manifest_blob=blob)) models.BlobManifest.objects.bulk_create(objs=thru, ignore_conflicts=True, batch_size=1000) tag = models.Tag(name=pk, tagged_manifest=manifest) diff --git a/pulp_container/constants.py b/pulp_container/constants.py index 9538f0fef..7da6b4f96 100644 --- a/pulp_container/constants.py +++ b/pulp_container/constants.py @@ -16,3 +16,4 @@ FOREIGN_BLOB_OCI="application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", ) EMPTY_BLOB = "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4" +BLOB_CONTENT_TYPE = "application/octet-stream"