Merge pull request #78 from dkliban/docker-push

Adds REST APIs needed to perform docker/podman push

CHANGES/5027.feature

@@ -0,0 +1 @@
+Added REST APIs for handling docker/podman push.

CHANGES/6776.bugfix

@@ -0,0 +1 @@
+Fixed 500 error when pulling by tag.

pulp_container/app/content.py

@@ -6,12 +6,14 @@
 
 registry = Registry()
 
-app.add_routes([web.get('/v2/', registry.serve_v2)])
-app.add_routes([web.get('/v2/_catalog', registry.list_repositories)])
-app.add_routes([web.get(r'/v2/{path:.+}/blobs/sha256:{digest:.+}', registry.get_by_digest)])
-app.add_routes([web.get(r'/v2/{path:.+}/manifests/sha256:{digest:.+}', registry.get_by_digest)])
-app.add_routes([web.get(r'/v2/{path:.+}/manifests/{tag_name}', registry.get_tag)])
-app.add_routes([web.get(r'/v2/{path:.+}/tags/list', registry.tags_list)])
+app.add_routes([web.get('/pulp/container/', registry.serve_v2)])
+app.add_routes([web.get('/pulp/container/_catalog', registry.list_repositories)])
+app.add_routes([web.get(r'/pulp/container/{path:.+}/blobs/sha256:{digest:.+}',
+                        registry.get_by_digest)])
+app.add_routes([web.get(r'/pulp/container/{path:.+}/manifests/sha256:{digest:.+}',
+                        registry.get_by_digest)])
+app.add_routes([web.get(r'/pulp/container/{path:.+}/manifests/{tag_name}', registry.get_tag)])
+app.add_routes([web.get(r'/pulp/container/{path:.+}/tags/list', registry.tags_list)])
 
 authorization_service = AuthorizationService()
 

pulp_container/app/migrations/0004_upload.py

@@ -0,0 +1,38 @@
+# Generated by Django 2.2.12 on 2020-05-19 20:57
+
+from django.db import migrations, models
+import django.db.models.deletion
+import pulp_container.app.models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0031_import_export_validate_params'),
+        ('container', '0003_oci_mediatype'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Upload',
+            fields=[
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('offset', models.BigIntegerField(default=0)),
+                ('file', models.FileField(max_length=255, null=True, upload_to=pulp_container.app.models.generate_filename)),
+                ('size', models.IntegerField(null=True)),
+                ('md5', models.CharField(max_length=32, null=True)),
+                ('sha1', models.CharField(max_length=40, null=True)),
+                ('sha224', models.CharField(max_length=56, null=True)),
+                ('sha256', models.CharField(max_length=64, null=True)),
+                ('sha384', models.CharField(max_length=96, null=True)),
+                ('sha512', models.CharField(max_length=128, null=True)),
+                ('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='uploads', to='core.Repository')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

pulp_container/app/models.py

@@ -1,19 +1,24 @@
+import hashlib
+import os
 import re
-
+import time
 from logging import getLogger
 
 from django.db import models
 from django.contrib.postgres import fields
 
 from pulpcore.plugin.download import DownloaderFactory
 from pulpcore.plugin.models import (
+    Artifact,
     Content,
+    BaseModel,
     Remote,
     Repository,
     RepositoryVersionDistribution
 )
 from pulpcore.plugin.repo_version_utils import remove_duplicates, validate_repo_version
 
+
 from . import downloaders
 from pulp_container.constants import MEDIA_TYPE
 
@@ -312,3 +317,63 @@ def get_repository_version(self):
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
+
+
+INCOMPLETE_EXT = '.part'
+
+
+def generate_filename(instance, filename):
+    """Method for generating upload file name"""
+    filename = os.path.join(instance.upload_dir, str(instance.pk) + INCOMPLETE_EXT)
+    return time.strftime(filename)
+
+
+class Upload(BaseModel):
+    """
+    Model for tracking Blob uploads.
+    """
+
+    repository = models.ForeignKey(
+        Repository, related_name='uploads', on_delete=models.CASCADE)
+
+    offset = models.BigIntegerField(default=0)
+
+    file = models.FileField(max_length=255, null=True, upload_to=generate_filename)
+
+    upload_dir = '/var/lib/pulp/tmp'
+
+    size = models.IntegerField(null=True)
+    md5 = models.CharField(max_length=32, null=True)
+    sha1 = models.CharField(max_length=40, null=True)
+    sha224 = models.CharField(max_length=56, null=True)
+    sha256 = models.CharField(max_length=64, null=True)
+    sha384 = models.CharField(max_length=96, null=True)
+    sha512 = models.CharField(max_length=128, null=True)
+
+    def append_chunk(self, chunk, chunk_size=None, save=True):
+        """Method for appending a chunk to a file."""
+        hashers = {}
+        for algorithm in Artifact.DIGEST_FIELDS:
+            hashers[algorithm] = getattr(hashlib, algorithm)()
+
+        self.file.close()
+        self.file.open(mode='ab')  # mode = append+binary
+        while True:
+            subchunk = chunk.read(2000000)
+            if not subchunk:
+                break
+            self.file.write(subchunk)
+            for algorithm in Artifact.DIGEST_FIELDS:
+                hashers[algorithm].update(subchunk)
+
+        if chunk_size is not None:
+            self.offset += chunk_size
+        elif hasattr(chunk, 'size'):
+            self.offset += chunk.size
+        else:
+            self.offset = self.file.size
+        if save:
+            self.save()
+        self.file.close()  # Flush
+        for algorithm in Artifact.DIGEST_FIELDS:
+            setattr(self, algorithm, hashers[algorithm].hexdigest())

pulp_container/app/registry.py

@@ -241,7 +241,7 @@ async def dispatch_tag(tag, response_headers):
 
         """
         try:
-            artifact = tag._artifacts.get()
+            artifact = tag.tagged_manifest._artifacts.get()
         except ObjectDoesNotExist:
             raise ArtifactNotFound(tag.name)
         else:

pulp_container/app/urls.py

@@ -0,0 +1,28 @@
+from django.conf.urls import url, include
+from rest_framework.routers import Route, SimpleRouter
+from pulp_container.app.viewsets import (
+    Blobs,
+    BlobUploads,
+    Manifests,
+    VersionView
+)
+
+router = SimpleRouter(trailing_slash=False)
+
+head_route = Route(
+    url=r'^{prefix}/{lookup}{trailing_slash}$',
+    mapping={'head': 'head'},
+    name='{basename}-detail',
+    detail=True,
+    initkwargs={'suffix': 'Instance'}
+)
+
+router.routes.append(head_route)
+router.register(r'^v2/(?P<path>.+)/blobs/uploads\/?', BlobUploads, basename='docker-upload')
+router.register(r'^v2/(?P<path>.+)/blobs', Blobs, basename='blobs')
+router.register(r'^v2/(?P<path>.+)/manifests', Manifests, basename='manifests')
+
+urlpatterns = [
+    url(r'^v2/$', VersionView.as_view()),
+    url(r'', include(router.urls))
+]