Create compose-images resource

This is a resource that works pretty much the same as compose-rpms. It
allows import compose images and getting the same data back.

The old import API is still available, but marked as deprecated.

Since the existing code did not store path for the images (just the
filename), there are two migrations to get to the desired state. After
applying the migrations, all data will be preserved, only the previously
existing images will have an empty path.

JIRA: PDC-911

pdc/apps/compose/fixtures/tests/compose_composeimage.json

@@ -1,26 +1,36 @@
 [
+  {
+    "model": "compose.Path",
+    "pk": 1,
+    "fields": {
+        "path": "path/to/images"
+    }
+  },
   {
     "model": "compose.ComposeImage",
     "pk": 1,
     "fields": {
         "variant_arch": 1,
-        "image": 1
+        "image": 1,
+        "path": 1
     }
   },
   {
     "model": "compose.ComposeImage",
     "pk": 2,
     "fields": {
         "variant_arch": 1,
-        "image": 2
+        "image": 2,
+        "path": 1
     }
   },
   {
     "model": "compose.ComposeImage",
     "pk": 3,
     "fields": {
         "variant_arch": 1,
-        "image": 3
+        "image": 3,
+        "path": 1
     }
   }
 ]

pdc/apps/compose/lib.py

@@ -200,31 +200,31 @@ def compose__import_images(request, release_id, composeinfo, image_manifest):
             var_arch_obj, created = models.VariantArch.objects.get_or_create(arch=arch_obj, variant=variant_obj)
 
             for i in im.images.get(variant.uid, {}).get(arch, []):
-                # TODO: handle properly
-                try:
-                    image = package_models.Image.objects.get(file_name=os.path.basename(i.path), sha256=i.checksums["sha256"])
-                except package_models.Image.DoesNotExist:
-                    image = package_models.Image()
-                image.file_name = os.path.basename(i.path)
-                image.image_format_id = package_models.ImageFormat.get_cached_id(i.format)
-                image.image_type_id = package_models.ImageType.get_cached_id(i.type)
-                image.disc_number = i.disc_number
-                image.disc_count = i.disc_count
-                image.arch = i.arch
-                image.mtime = i.mtime
-                image.size = i.size
-                image.bootable = i.bootable
-                image.implant_md5 = i.implant_md5
-                image.volume_id = i.volume_id
-                image.md5 = i.checksums.get("md5", None)
-                image.sha1 = i.checksums.get("sha1", None)
-                image.sha256 = i.checksums.get("sha256", None)
-                image.save()
-
-                # TODO: path to ComposeImage
+                path, file_name = os.path.split(i.path)
+                path_id = models.Path.get_cached_id(path, create=True)
+
+                image, _ = package_models.Image.objects.get_or_create(
+                    file_name=file_name, sha256=i.checksums["sha256"],
+                    defaults={
+                        'image_format_id': package_models.ImageFormat.get_cached_id(i.format),
+                        'image_type_id': package_models.ImageType.get_cached_id(i.type),
+                        'disc_number': i.disc_number,
+                        'disc_count': i.disc_count,
+                        'arch': i.arch,
+                        'mtime': i.mtime,
+                        'size': i.size,
+                        'bootable': i.bootable,
+                        'implant_md5': i.implant_md5,
+                        'volume_id': i.volume_id,
+                        'md5': i.checksums.get("md5", None),
+                        'sha1': i.checksums.get("sha1", None),
+                    }
+                )
+
                 mi, created = models.ComposeImage.objects.get_or_create(
                     variant_arch=var_arch_obj,
-                    image=image)
+                    image=image,
+                    path_id=path_id)
                 imported_images += 1
 
     for obj in add_to_changelog:

pdc/apps/compose/migrations/0004_auto_20150819_0826.py

@@ -0,0 +1,29 @@
+#
+# Copyright (c) 2015 Red Hat
+# Licensed under The MIT License (MIT)
+# http://opensource.org/licenses/MIT
+#
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('compose', '0003_auto_20150610_1338'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='composeimage',
+            name='path',
+            field=models.ForeignKey(to='compose.Path', null=True),
+        ),
+        migrations.AlterField(
+            model_name='path',
+            name='path',
+            field=models.CharField(unique=True, max_length=4096, blank=True),
+        ),
+    ]

pdc/apps/compose/migrations/0005_auto_20150819_0827.py

@@ -0,0 +1,32 @@
+#
+# Copyright (c) 2015 Red Hat
+# Licensed under The MIT License (MIT)
+# http://opensource.org/licenses/MIT
+#
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+def set_empty_path(apps, schema_editor):
+    path, _ = apps.get_model('compose', 'Path').objects.get_or_create(path='')
+    for compose_image in apps.get_model('compose', 'ComposeImage').objects.filter(path=None):
+        compose_image.path = path
+        compose_image.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('compose', '0004_auto_20150819_0826'),
+    ]
+
+    operations = [
+        migrations.RunPython(set_empty_path),
+        migrations.AlterField(
+            model_name='composeimage',
+            name='path',
+            field=models.ForeignKey(to='compose.Path'),
+        ),
+    ]

pdc/apps/compose/models.py

@@ -218,7 +218,7 @@ class Path(models.Model):
     Class representing a path in compose. The same path can be reused in
     multiple places (e.g. in more RPMs).
     """
-    path = models.CharField(max_length=4096, unique=True)
+    path = models.CharField(max_length=4096, unique=True, blank=True)
 
     def __unicode__(self):
         return unicode(self.path)
@@ -499,6 +499,7 @@ def update_object(klass, action, release, data):
 class ComposeImage(models.Model):
     variant_arch        = models.ForeignKey(VariantArch, db_index=True)
     image               = models.ForeignKey("package.Image", db_index=True)
+    path                = models.ForeignKey(Path)
 
     class Meta:
         unique_together = (

pdc/apps/compose/tests.py

@@ -783,11 +783,13 @@ def setUp(self):
             self.compose_info = json.loads(f.read())
         with open('pdc/apps/compose/fixtures/tests/image-manifest.json', 'r') as f:
             self.manifest = json.loads(f.read())
+        self.client.post(reverse('releaseimportcomposeinfo-list'),
+                         self.compose_info, format='json')
+        # Caching ids makes it faster, but the cache needs to be cleared for each test.
+        models.Path.CACHE = {}
 
-    def test_import_images(self):
-        response = self.client.post(reverse('releaseimportcomposeinfo-list'),
-                                    self.compose_info, format='json')
-        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+    def test_import_images_by_deprecated_api(self):
+        # TODO: remove this test after next release
         response = self.client.post(reverse('composeimportimages-list'),
                                     {'image_manifest': self.manifest,
                                      'release_id': 'tp-1.0',
@@ -800,6 +802,31 @@ def test_import_images(self):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.data.get('count'), 4)
 
+    def test_import_images(self):
+        response = self.client.post(reverse('composeimage-list'),
+                                    {'image_manifest': self.manifest,
+                                     'release_id': 'tp-1.0',
+                                     'composeinfo': self.compose_info},
+                                    format='json')
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+        self.assertNumChanges([11, 5])
+        self.assertEqual(models.ComposeImage.objects.count(), 4)
+        response = self.client.get(reverse('image-list'), {'compose': 'TP-1.0-20150310.0'})
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data.get('count'), 4)
+
+    def test_import_and_retrieve_images(self):
+        response = self.client.post(reverse('composeimage-list'),
+                                    {'image_manifest': self.manifest,
+                                     'release_id': 'tp-1.0',
+                                     'composeinfo': self.compose_info},
+                                    format='json')
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+        response = self.client.get(reverse('composeimage-detail', args=['TP-1.0-20150310.0']))
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.maxDiff = None
+        self.assertDictEqual(dict(response.data), self.manifest)
+
 
 class RPMMappingAPITestCase(APITestCase):
     fixtures = [

pdc/apps/compose/views.py

@@ -8,6 +8,7 @@
 import os.path
 
 from productmd.rpms import Rpms
+from productmd.images import Images, Image
 
 from django.conf import settings
 from kobo.django.views.generic import DetailView, SearchView
@@ -783,16 +784,19 @@ def update(self, request, **kwargs):
         return Response(changes)
 
 
-class ComposeImportImagesView(StrictQueryParamMixin, viewsets.GenericViewSet):
+class ComposeImageView(StrictQueryParamMixin,
+                       viewsets.GenericViewSet):
     queryset = ComposeImage.objects.none()  # Required for permissions
+    lookup_field = 'compose_id'
+    lookup_value_regex = '[^/]+'
 
     def create(self, request):
         """
         Import images.
 
         __Method__: POST
 
-        __URL__: `/rpc/compose/import-images/`
+        __URL__: `/compose-images/`
 
         __Data__:
 
@@ -824,6 +828,67 @@ def create(self, request):
         lib.compose__import_images(request, data['release_id'], data['composeinfo'], data['image_manifest'])
         return Response(status=201)
 
+    def retrieve(self, request, **kwargs):
+        """
+        **Method**: `GET`
+
+        **URL**: `/compose-images/{compose_id}/`
+
+        This API end-point allows retrieving RPM manifest for a given compose.
+        It will return the exact same data as was imported.
+        """
+        compose = get_object_or_404(Compose, compose_id=kwargs['compose_id'])
+        cimages = ComposeImage.objects.filter(variant_arch__variant__compose=compose)
+
+        manifest = Images()
+        manifest.compose.date = compose.compose_date.strftime('%Y%m%d')
+        manifest.compose.id = compose.compose_id
+        manifest.compose.respin = compose.compose_respin
+        manifest.compose.type = compose.compose_type.name
+
+        for cimage in cimages:
+            im = Image(None)
+            im.path = os.path.join(cimage.path.path, cimage.image.file_name)
+            im.arch = cimage.image.arch
+            im.bootable = cimage.image.bootable
+            im.mtime = cimage.image.mtime
+            im.size = cimage.image.size
+            im.volume_id = cimage.image.volume_id
+            im.type = cimage.image.image_type.name
+            im.format = cimage.image.image_format.name
+            im.arch = cimage.image.arch
+            im.disc_number = cimage.image.disc_number
+            im.disc_count = cimage.image.disc_count
+            im.checksums = {'sha256': cimage.image.sha256}
+            if cimage.image.md5:
+                im.checksums['md5'] = cimage.image.md5
+            if cimage.image.sha1:
+                im.checksums['sha1'] = cimage.image.sha1
+            im.implant_md5 = cimage.image.implant_md5
+            manifest.add(cimage.variant_arch.variant.variant_uid, cimage.variant_arch.arch.name, im)
+
+        return Response(manifest.serialize({}))
+
+
+class ComposeImportImagesView(StrictQueryParamMixin, viewsets.GenericViewSet):
+    # TODO: remove this class after next release
+    queryset = ComposeImage.objects.none()  # Required for permissions
+
+    def create(self, request):
+        """
+        This end-point is deprecated. Use
+        [/compose-images/](/rest_api/v1/compose-images/) instead.
+        """
+        data = request.data
+        errors = {}
+        for key in ('release_id', 'composeinfo', 'image_manifest'):
+            if key not in data:
+                errors[key] = ["This field is required"]
+        if errors:
+            return Response(status=400, data=errors)
+        lib.compose__import_images(request, data['release_id'], data['composeinfo'], data['image_manifest'])
+        return Response(status=201)
+
 
 class ReleaseOverridesRPMViewSet(StrictQueryParamMixin,
                                  mixins.ListModelMixin,

pdc/routers.py

@@ -74,6 +74,7 @@
                 compose_views.ComposeRPMMappingView,
                 base_name='composerpmmapping')
 router.register(r'compose-rpms', compose_views.ComposeRPMView)
+router.register(r'compose-images', compose_views.ComposeImageView)
 
 router.register(r'rpc/release/import-from-composeinfo',
                 release_views.ReleaseImportView,