Support upload tarball with incomplete images.

closes #3497

docs/user-guide/recipes.rst

@@ -378,6 +378,13 @@ with the directory contents created by ``skopeo copy``::
 
     $ cd existingemptydirectory/ && tar -cvf ../image-name.tar * && cd ..
 
+
+.. note::
+
+    The tarball is only required to contain blob files for layers referenced in the manifest if they
+    are not already contained in the specified repository. Blob files included in the tarball that are
+    already contained in the repository will be ignored.
+
 Then create a Pulp repository and run an upload command with ``pulp-admin``::
 
     $ pulp-admin docker repo create --repo-id=schema2

docs/user-guide/release-notes/3.2.x.rst

@@ -0,0 +1,9 @@
+3.2 Release Notes
+=================
+
+3.2.0
+-----
+
+Blob files for layers referenced in the manifest of an uploaded tarball no longer need to be
+included in the tarball if they are already contained in the specified repository. Blob files
+included in the tarball that are already contained in the repository will be ignored.

docs/user-guide/release-notes/index.rst

@@ -6,6 +6,7 @@ Contents:
 .. toctree::
    :maxdepth: 2
 
+   3.2.x
    3.1.x
    3.0.x
    2.4.x

plugins/pulp_docker/plugins/importers/upload.py

@@ -25,7 +25,7 @@
 import stat
 import tarfile
 
-from mongoengine import NotUniqueError
+from mongoengine import NotUniqueError, Q
 
 from pulp.plugins.util.publish_step import PluginStep, GetLocalUnitsStep
 from pulp.server.db import model as pulp_models
@@ -411,23 +411,26 @@ def process_main(self, item=None):
                 checksum_type = "sha256"
             blob_src_path = os.path.join(self.get_working_dir(), checksum + '.tar')
             try:
-                fobj = open(blob_src_path)
+                with open(blob_src_path) as fobj:
+                    try:
+                        verification.verify_checksum(fobj, checksum_type, checksum)
+                    except verification.VerificationException:
+                        raise PulpCodedValidationException(
+                            error_code=error_codes.DKR1017,
+                            checksum_type=checksum_type,
+                            checksum=checksum)
             except IOError:
+                blobs = repository.find_repo_content_units(
+                    self.get_repo().repo_obj,
+                    units_q=Q(digest=item.digest),
+                    limit=1)
+                if tuple(blobs):
+                    return
                 raise PulpCodedValidationException(
                     error_code=error_codes.DKR1018,
                     layer=os.path.basename(blob_src_path))
-            try:
-                verification.verify_checksum(fobj, checksum_type, checksum)
-            except verification.VerificationException:
-                raise PulpCodedValidationException(
-                    error_code=error_codes.DKR1017,
-                    checksum_type=checksum_type,
-                    checksum=checksum)
-            fobj.close()
-
             blob_dest_path = os.path.join(self.get_working_dir(), item.digest)
             os.rename(blob_src_path, blob_dest_path)
-
         item.set_storage_path(item.digest)
         try:
             item.save_and_import_content(os.path.join(self.get_working_dir(), item.digest))

plugins/test/unit/plugins/importers/test_upload.py

@@ -137,6 +137,7 @@ def test_AddUnits_error_bad_checksum(self, _repo_controller, _Manifest_save, _Bl
             str(ctx.exception))
 
     def test_AddUnits_error_missing_layer(self, _repo_controller, _Manifest_save, _Blob_save):
+        _repo_controller.find_repo_content_units.return_value = ()
         # This is where we will untar the image
         step_work_dir = os.path.join(self.work_dir, "working_dir")
         os.makedirs(step_work_dir)
@@ -168,6 +169,41 @@ def test_AddUnits_error_missing_layer(self, _repo_controller, _Manifest_save, _B
             "Layer this-is-missing.tar is not present in the image",
             str(ctx.exception))
 
+    def test_AddUnits_existing_layer(self, _repo_controller, _Manifest_save, _Blob_save):
+        _repo_controller.find_repo_content_units.return_value = (mock.Mock(),)
+        # This is where we will untar the image
+        step_work_dir = os.path.join(self.work_dir, "working_dir")
+        os.makedirs(step_work_dir)
+
+        img, layers = self._create_image()
+        manifest_data = dict(layers=[dict(digest=x['digest'],
+                                          mediaType="ignored")
+                                     for x in layers],
+                             config=dict(digest="abc"),
+                             schemaVersion=2)
+        units = [
+            models.Manifest.from_json(json.dumps(manifest_data), digest="012"),
+        ]
+        units.extend(models.Blob(digest="sha256:%s" % x['digest']) for x in layers)
+        # This layer not in the tarball.
+        units.append(models.Blob(digest="sha256:this-already-in-the-repository"))
+
+        parent = mock.MagicMock()
+        parent.configure_mock(file_path=img, parent=None)
+        parent.v2_step_get_local_units.units_to_download = units
+        step = upload.AddUnits(step_type=constants.UPLOAD_STEP_SAVE,
+                               working_dir=step_work_dir)
+        step.parent = parent
+        step.process_lifecycle()
+        # Make sure associate_single_unit got called
+        repo_obj = parent.get_repo.return_value.repo_obj
+        self.assertEquals(
+            [mock.call(repo_obj, x) for x in units[:-1]],
+            _repo_controller.associate_single_unit.call_args_list)
+        self.assertEquals(
+            units[0],
+            parent.uploaded_unit)
+
     @mock.patch("pulp_docker.plugins.importers.upload.models.Manifest.objects")
     @mock.patch("pulp_docker.plugins.importers.upload.models.Tag.objects")
     def test_AddTags(self, _Tag_objects, _Manifest_objects, _repos,