Add a unit for the new Docker metadata type.

The Docker v2 API introduces a new Image Manifest type, described here:

https://github.com/docker/distribution/blob/release/2.0/docs/spec/manifest-v2-1.md

https://pulp.plan.io/issues/967

fixes #967

.gitignore

@@ -55,3 +55,6 @@ docs/*/_build/
 
 # pycharm
 .idea/
+
+# vim
+*.swp

common/pulp_docker/common/models.py

@@ -1,3 +1,8 @@
+"""
+This module contains common model objects that are used to describe the data types used in the
+pulp_docker plugins.
+"""
+import json
 import os
 
 from pulp_docker.common import constants
@@ -50,3 +55,105 @@ def unit_metadata(self):
             'parent_id': self.parent_id,
             'size': self.size
         }
+
+
+class DockerManifest(object):
+    """
+    This model represents a Docker v2, Schema 1 Image Manifest, as described here:
+
+    https://github.com/docker/distribution/blob/release/2.0/docs/spec/manifest-v2-1.md
+    """
+    TYPE_ID = 'docker_manifest'
+
+    def __init__(self, name, tag, architecture, digest, fs_layers, history, schema_version,
+                 signatures):
+        """
+        Initialize the DockerManifest model with the given attributes. See the class docblock above
+        for a link to the Docker documentation that covers these attributes. Note that this class
+        attempts to follow Python naming guidelines for the class attributes, while allowing
+        Docker's camelCase names for the inner values on dictionaries.
+
+        :param name:           The name of the image's repository
+        :type  name:           basestring
+        :param tag:            The image's tag
+        :type  tag:            basestring
+        :param architecture:   The host architecture on which the image is intended to run
+        :type  architecture:   basestring
+        :param digest:         The content digest of the manifest, as described at
+                               https://docs.docker.com/registry/spec/api/#content-digests
+        :type  digest:         basestring
+        :param fs_layers:      A list of dictionaries. Each dictionary contains one key-value pair
+                               that represents a layer of the image. The key is blobSum, and the
+                               value is the digest of the referenced layer. See the documentation
+                               referenced in the class docblock for more information.
+        :type  fs_layers:      list
+        :param history:        This is a list of unstructured historical data for v1 compatibility.
+                               Each member is a dictionary with a "v1Compatibility" key that indexes
+                               a string.
+        :type  history:        list
+        :param schema_version: The image manifest schema that this image follows
+        :type  schema_version: int
+        :param signatures:     A list of cryptographic signatures on the image. See the
+                               documentation in the in this class's docblock for information about
+                               its formatting.
+        :type  signatures:     list
+        """
+        self.name = name
+        self.tag = tag
+        self.architecture = architecture
+        self.digest = digest
+        self.fs_layers = fs_layers
+        self.history = history
+        self.signatures = signatures
+
+        if schema_version != 1:
+            raise ValueError(
+                "The DockerManifest class only supports Docker v2, Schema 1 manifests.")
+        self.schema_version = schema_version
+
+    @classmethod
+    def from_json(cls, manifest_json, digest):
+        """
+        Construct and return a DockerManifest from the given JSON document.
+
+        :param manifest_json: A JSON document describing a DockerManifest object as defined by the
+                              Docker v2, Schema 1 Image Manifest documentation.
+        :type  manifest_json: basestring
+        :param digest:        The content digest of the manifest, as described at
+                              https://docs.docker.com/registry/spec/api/#content-digests
+        :type  digest:        basestring
+
+        :return:              An initialized DockerManifest object
+        :rtype:               pulp_docker.common.models.DockerManifest
+        """
+        manifest = json.loads(manifest_json)
+        return cls(
+            name=manifest['name'], tag=manifest['tag'], architecture=manifest['architecture'],
+            digest=digest, fs_layers=manifest['fsLayers'], history=manifest['history'],
+            schema_version=manifest['schemaVersion'], signatures=manifest['signatures'])
+
+    @property
+    def to_json(self):
+        """
+        Return a JSON document that represents the DockerManifest object.
+
+        :return: A JSON document in the Docker v2, Schema 1 Image Manifest format
+        :rtype:  basestring
+        """
+        manifest = {
+            'name': self.name, 'tag': self.tag, 'architecture': self.architecture,
+            'fsLayers': self.fs_layers, 'history': self.history,
+            'schemaVersion': self.schema_version, 'signatures': self.signatures}
+        return json.dumps(manifest)
+
+    @property
+    def unit_key(self):
+        """
+        Return the manifest's unit key. The unit key consists of the name, tag, architecture, and
+        fs_layers attributes as described in the __init__() method above.
+
+        :return: unit key
+        :rtype:  dict
+        """
+        return {'name': self.name, 'tag': self.tag, 'architecture': self.architecture,
+                'digest': self.digest}

common/test/data/example_docker_v2_manifest.json

@@ -0,0 +1,44 @@
+{
+   "name": "hello-world",
+   "tag": "latest",
+   "architecture": "amd64",
+   "fsLayers": [
+      {
+         "blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"
+      },
+      {
+         "blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"
+      },
+      {
+         "blobSum": "sha256:cc8567d70002e957612902a8e985ea129d831ebe04057d88fb644857caa45d11"
+      },
+      {
+         "blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"
+      }
+   ],
+   "history": [
+      {
+         "v1Compatibility": "{\"id\":\"e45a5af57b00862e5ef5782a9925979a02ba2b12dff832fd0991335f4a11e5c5\",\"parent\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"created\":\"2014-12-31T22:57:59.178729048Z\",\"container\":\"27b45f8fb11795b52e9605b686159729b0d9ca92f76d40fb4f05a62e19c46b4f\",\"container_config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [/hello]\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"docker_version\":\"1.4.1\",\"config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/hello\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
+      },
+      {
+         "v1Compatibility": "{\"id\":\"e45a5af57b00862e5ef5782a9925979a02ba2b12dff832fd0991335f4a11e5c5\",\"parent\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"created\":\"2014-12-31T22:57:59.178729048Z\",\"container\":\"27b45f8fb11795b52e9605b686159729b0d9ca92f76d40fb4f05a62e19c46b4f\",\"container_config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [/hello]\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"docker_version\":\"1.4.1\",\"config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/hello\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
+      }
+   ],
+   "schemaVersion": 1,
+   "signatures": [
+      {
+         "header": {
+            "jwk": {
+               "crv": "P-256",
+               "kid": "OD6I:6DRK:JXEJ:KBM4:255X:NSAA:MUSF:E4VM:ZI6W:CUN2:L4Z6:LSF4",
+               "kty": "EC",
+               "x": "3gAwX48IQ5oaYQAYSxor6rYYc_6yjuLCjtQ9LUakg4A",
+               "y": "t72ge6kIA1XOjqjVoEOiPPAURltJFBMGDSQvEGVB010"
+            },
+            "alg": "ES256"
+         },
+         "signature": "XREm0L8WNn27Ga_iE_vRnTxVMhhYY0Zst_FfkKopg6gWSoTOZTuW4rK0fg_IqnKkEKlbD83tD46LKEGi5aIVFg",
+         "protected": "eyJmb3JtYXRMZW5ndGgiOjY2MjgsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wNC0wOFQxODo1Mjo1OVoifQ"
+      }
+   ]
+}

common/test/unit/test_models.py

@@ -1,3 +1,9 @@
+"""
+This modules contains tests for pulp_docker.common.models.
+"""
+import json
+import math
+import os
 import unittest
 
 from pulp_docker.common import models
@@ -27,3 +33,124 @@ def test_metadata(self):
 
         self.assertEqual(metadata.get('parent_id'), 'xyz')
         self.assertEqual(metadata.get('size'), 1024)
+
+
+class TestDockerManifest(unittest.TestCase):
+    """
+    This class contains tests for the DockerManifest class.
+    """
+    def test___init__(self):
+        """
+        Assert correct operation of the __init__() method.
+        """
+        name = 'name'
+        tag = 'tag'
+        architecture = 'x86_65'  # it's one better
+        digest = 'sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b'
+        fs_layers = [{'layer_1': 'rsum:jsf'}]
+        history = [{'v1Compatibility': 'not sure what goes here but something does'}]
+        schema_version = 1
+        signatures = [{'some': 'signature'}]
+
+        m = models.DockerManifest(name, tag, architecture, digest, fs_layers, history,
+                                  schema_version, signatures)
+
+        self.assertEqual(m.name, name)
+        self.assertEqual(m.tag, tag)
+        self.assertEqual(m.architecture, architecture)
+        self.assertEqual(m.digest, digest)
+        self.assertEqual(m.fs_layers, fs_layers)
+        self.assertEqual(m.history, history)
+        self.assertEqual(m.signatures, signatures)
+        self.assertEqual(m.schema_version, schema_version)
+
+    def test___init___bad_schema(self):
+        """
+        Assert correct operation of the __init__() method with an invalid (i.e., != 1) schema
+        version.
+        """
+        name = 'name'
+        tag = 'tag'
+        architecture = 'x86_65'  # it's one better
+        digest = 'sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b'
+        fs_layers = [{'layer_1': 'rsum:jsf'}]
+        history = [{'v1Compatibility': 'not sure what goes here but something does'}]
+        schema_version = math.pi
+        signatures = [{'some': 'signature'}]
+
+        self.assertRaises(ValueError, models.DockerManifest, name, tag, architecture, digest,
+                          fs_layers, history, schema_version, signatures)
+
+    def test_from_json(self):
+        """
+        Assert correct operation of the from_json class method.
+        """
+        digest = 'sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b'
+        example_manifest_path = os.path.join(os.path.dirname(__file__), '..', 'data',
+                                             'example_docker_v2_manifest.json')
+        with open(example_manifest_path) as manifest_file:
+            manifest = manifest_file.read()
+
+        m = models.DockerManifest.from_json(manifest, digest)
+
+        self.assertEqual(m.name, 'hello-world')
+        self.assertEqual(m.tag, 'latest')
+        self.assertEqual(m.architecture, 'amd64')
+        self.assertEqual(m.digest, digest)
+        self.assertEqual(m.schema_version, 1)
+        # We will just spot check the following attributes, as they are complex data structures
+        self.assertEqual(len(m.fs_layers), 4)
+        self.assertEqual(
+            m.fs_layers[1],
+            {"blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"})
+        self.assertEqual(len(m.history), 2)
+        self.assertTrue('],\"Image\":\"31cbccb51277105ba3ae35ce' in m.history[0]['v1Compatibility'])
+        self.assertEqual(len(m.signatures), 1)
+        self.assertTrue('XREm0L8WNn27Ga_iE_vRnTxVMhhYY0Zst_FfkKopg' in m.signatures[0]['signature'])
+
+    def test_to_json(self):
+        """
+        Assert correct operation from the to_json() property.
+        """
+        name = 'name'
+        tag = 'tag'
+        architecture = 'x86_65'  # it's one better
+        digest = 'sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b'
+        fs_layers = [{'layer_1': 'rsum:jsf'}]
+        history = [{'v1Compatibility': 'not sure what goes here but something does'}]
+        schema_version = 1
+        signatures = [{'some': 'signature'}]
+        m = models.DockerManifest(name, tag, architecture, digest, fs_layers, history,
+                                  schema_version, signatures)
+
+        j = m.to_json
+
+        # In order to assert that j is correct, we'll load it back to a dictionary for comparison
+        data = json.loads(j)
+        self.assertEqual(data['name'], name)
+        self.assertEqual(data['tag'], tag)
+        self.assertEqual(data['architecture'], architecture)
+        self.assertEqual(data['fsLayers'], fs_layers)
+        self.assertEqual(data['history'], history)
+        self.assertEqual(data['schemaVersion'], schema_version)
+        self.assertEqual(data['signatures'], signatures)
+
+    def test_unit_key(self):
+        """
+        Assert correct operation of the unit_key property.
+        """
+        name = 'name'
+        tag = 'tag'
+        architecture = 'x86_65'  # it's one better
+        digest = 'sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b'
+        fs_layers = [{'layer_1': 'rsum:jsf'}]
+        history = [{'v1Compatibility': 'not sure what goes here but something does'}]
+        schema_version = 1
+        signatures = [{'some': 'signature'}]
+        m = models.DockerManifest(name, tag, architecture, digest, fs_layers, history,
+                                  schema_version, signatures)
+
+        unit_key = m.unit_key
+
+        self.assertEqual(unit_key, {'name': name, 'tag': tag, 'architecture': architecture,
+                                    'digest': digest})

plugins/types/docker.json

@@ -5,5 +5,12 @@
         "description": "Docker Image",
         "unit_key": ["image_id"],
         "search_indexes": []
+    },
+    {
+        "id": "docker_manifest",
+        "display_name": "Docker Manifest",
+        "description": "Docker Manifest",
+        "unit_key": ["name", "tag", "architecture", "digest"],
+        "search_indexes": []
     }
 ]}