Merge "Support new image copied from external storage."

bin/glance

@@ -235,13 +235,21 @@ EXAMPLES
             print 'Found non-settable field %s. Removing.' % field
             fields.pop(field)
 
-    if 'location' in fields.keys():
-        image_meta['location'] = fields.pop('location')
+    def _external_source(fields, image_data):
+        source = None
+        features = {}
+        if 'location' in fields.keys():
+            source = fields.pop('location')
+            image_meta['location'] = source
+        elif 'copy_from' in fields.keys():
+            source = fields.pop('copy_from')
+            features['x-glance-api-copy-from'] = source
+        return source, features
 
     # We need either a location or image data/stream to add...
-    image_location = image_meta.get('location')
+    location, features = _external_source(fields, image_meta)
     image_data = None
-    if not image_location:
+    if not location:
         # Grab the image data stream from stdin or redirect,
         # otherwise error out
         image_data = sys.stdin
@@ -251,7 +259,8 @@ EXAMPLES
 
     if not options.dry_run:
         try:
-            image_meta = c.add_image(image_meta, image_data)
+            image_meta = c.add_image(image_meta, image_data,
+                                     features=features)
             image_id = image_meta['id']
             print "Added new image with ID: %s" % image_id
             if options.verbose:
@@ -278,9 +287,17 @@ EXAMPLES
             return FAILURE
     else:
         print "Dry run. We would have done the following:"
+
+        def _dump(dict):
+            for k, v in sorted(dict.items()):
+                print " %(k)30s => %(v)s" % locals()
+
         print "Add new image with metadata:"
-        for k, v in sorted(image_meta.items()):
-            print " %(k)30s => %(v)s" % locals()
+        _dump(image_meta)
+
+        if features:
+            print "with features enabled:"
+            _dump(features)
 
     return SUCCESS
 
@@ -299,7 +316,8 @@ to Glance that represents the metadata for an image.
 Field names that can be specified:
 
 name                A name for the image.
-location            The location of the image.
+location            An external location to serve out from.
+copy_from           An external location (HTTP, S3 or Swift URI) to copy from.
 is_public           If specified, interpreted as a boolean value
                     and sets or unsets the image's availability to the public.
 protected           If specified, interpreted as a boolean value

doc/source/glance.rst

@@ -260,13 +260,20 @@ To upload an EC2 tarball VM image with an associated property (e.g., distro)::
      container_format=ovf disk_format=raw \
      distro="ubuntu 10.10" < /root/maverick-server-uec-amd64.tar.gz
 
-To upload an EC2 tarball VM image from a URL::
+To reference an EC2 tarball VM image available at an external URL::
 
-  $> glance add name="uubntu-10.04-amd64" is_public=true \
+  $> glance add name="ubuntu-10.04-amd64" is_public=true \
      container_format=ovf disk_format=raw \
      location="http://uec-images.ubuntu.com/lucid/current/\
      lucid-server-uec-amd64.tar.gz"
 
+To upload a copy of that same EC2 tarball VM image::
+
+  $> glance add name="ubuntu-10.04-amd64" is_public=true \
+     container_format=ovf disk_format=raw \
+     copy_from="http://uec-images.ubuntu.com/lucid/current/\
+     lucid-server-uec-amd64.tar.gz"
+
 To upload a qcow2 image::
 
   $> glance add name="ubuntu-11.04-amd64" is_public=true \

glance/api/v1/images.py

@@ -226,6 +226,23 @@ def meta(self, req, id):
             'image_meta': image_meta
         }
 
+    @staticmethod
+    def _copy_from(req):
+        return req.headers.get('x-glance-api-copy-from')
+
+    @staticmethod
+    def _external_source(image_meta, req):
+        return image_meta.get('location', Controller._copy_from(req))
+
+    @staticmethod
+    def _get_from_store(where):
+        try:
+            image_data, image_size = get_from_backend(where)
+        except exception.NotFound, e:
+            raise HTTPNotFound(explanation="%s" % e)
+        image_size = int(image_size) if image_size else None
+        return image_data, image_size
+
     def show(self, req, id):
         """
         Returns an iterator that can be used to retrieve an image's
@@ -239,16 +256,9 @@ def show(self, req, id):
         self._enforce(req, 'get_image')
         image_meta = self.get_active_image_meta_or_404(req, id)
 
-        def get_from_store(image_meta):
-            try:
-                location = image_meta['location']
-                image_data, image_size = get_from_backend(location)
-                image_meta["size"] = image_size or image_meta["size"]
-            except exception.NotFound, e:
-                raise HTTPNotFound(explanation="%s" % e)
-            return image_data
-
-        image_iterator = get_from_store(image_meta)
+        image_iterator, size = self._get_from_store(image_meta['location'])
+        image_meta['size'] = size or image_meta['size']
+
         del image_meta['location']
         return {
             'image_iterator': image_iterator,
@@ -263,11 +273,12 @@ def _reserve(self, req, image_meta):
 
         :param req: The WSGI/Webob Request object
         :param id: The opaque image identifier
+        :param image_meta: The image metadata
 
         :raises HTTPConflict if image already exists
         :raises HTTPBadRequest if image metadata is not valid
         """
-        location = image_meta.get('location')
+        location = self._external_source(image_meta, req)
         if location:
             store = get_store_from_location(location)
             # check the store exists before we hit the registry, but we
@@ -278,9 +289,9 @@ def _reserve(self, req, image_meta):
             image_meta['size'] = image_meta.get('size', 0) \
                                  or get_size_from_backend(location)
         else:
-            # Ensure that the size attribute is set to zero for uploadable
-            # images (if not provided). The size will be set to a non-zero
-            # value during upload
+            # Ensure that the size attribute is set to zero for directly
+            # uploadable images (if not provided). The size will be set
+            # to a non-zero value during upload
             image_meta['size'] = image_meta.get('size', 0)
 
         image_meta['status'] = 'queued'
@@ -317,13 +328,30 @@ def _upload(self, req, image_meta):
         :raises HTTPConflict if image already exists
         :retval The location where the image was stored
         """
-        try:
-            req.get_content_type('application/octet-stream')
-        except exception.InvalidContentType:
-            self._safe_kill(req, image_meta['id'])
-            msg = _("Content-Type must be application/octet-stream")
-            logger.error(msg)
-            raise HTTPBadRequest(explanation=msg)
+
+        copy_from = self._copy_from(req)
+        if copy_from:
+            image_data, image_size = self._get_from_store(copy_from)
+            image_meta['size'] = image_size or image_meta['size']
+        else:
+            try:
+                req.get_content_type('application/octet-stream')
+            except exception.InvalidContentType:
+                self._safe_kill(req, image_meta['id'])
+                msg = _("Content-Type must be application/octet-stream")
+                logger.error(msg)
+                raise HTTPBadRequest(explanation=msg)
+
+            image_data = req.body_file
+
+            if req.content_length:
+                image_size = int(req.content_length)
+            elif 'x-image-meta-size' in req.headers:
+                image_size = int(req.headers['x-image-meta-size'])
+            else:
+                logger.debug(_("Got request with no content-length and no "
+                               "x-image-meta-size header"))
+                image_size = 0
 
         store_name = req.headers.get('x-image-meta-store',
                                      self.conf.default_store)
@@ -337,14 +365,6 @@ def _upload(self, req, image_meta):
         try:
             logger.debug(_("Uploading image data for image %(image_id)s "
                          "to %(store_name)s store"), locals())
-            if req.content_length:
-                image_size = int(req.content_length)
-            elif 'x-image-meta-size' in req.headers:
-                image_size = int(req.headers['x-image-meta-size'])
-            else:
-                logger.debug(_("Got request with no content-length and no "
-                               "x-image-meta-size header"))
-                image_size = 0
 
             if image_size > IMAGE_SIZE_CAP:
                 max_image_size = IMAGE_SIZE_CAP
@@ -355,7 +375,7 @@ def _upload(self, req, image_meta):
                 raise HTTPBadRequest(msg, request=request)
 
             location, size, checksum = store.add(image_meta['id'],
-                                                 req.body_file,
+                                                 image_data,
                                                  image_size)
 
             # Verify any supplied checksum value matches checksum
@@ -505,19 +525,27 @@ def _upload_and_activate(self, req, image_meta):
 
     def create(self, req, image_meta, image_data):
         """
-        Adds a new image to Glance. Three scenarios exist when creating an
+        Adds a new image to Glance. Four scenarios exist when creating an
         image:
 
-        1. If the image data is available for upload, create can be passed the
-           image data as the request body and the metadata as the request
-           headers. The image will initially be 'queued', during upload it
-           will be in the 'saving' status, and then 'killed' or 'active'
-           depending on whether the upload completed successfully.
+        1. If the image data is available directly for upload, create can be
+           passed the image data as the request body and the metadata as the
+           request headers. The image will initially be 'queued', during
+           upload it will be in the 'saving' status, and then 'killed' or
+           'active' depending on whether the upload completed successfully.
+
+        2. If the image data exists somewhere else, you can upload indirectly
+           from the external source using the x-glance-api-copy-from header.
+           Once the image is uploaded, the external store is not subsequently
+           consulted, i.e. the image content is served out from the configured
+           glance image store.  State transitions are as for option #1.
 
-        2. If the image data exists somewhere else, you can pass in the source
-           using the x-image-meta-location header
+        3. If the image data exists somewhere else, you can reference the
+           source using the x-image-meta-location header. The image content
+           will be served out from the external store, i.e. is never uploaded
+           to the configured glance image store.
 
-        3. If the image data is not available yet, but you'd like reserve a
+        4. If the image data is not available yet, but you'd like reserve a
            spot for it, you can omit the data and a record will be created in
            the 'queued' state. This exists primarily to maintain backwards
            compatibility with OpenStack/Rackspace API semantics.
@@ -547,7 +575,7 @@ def create(self, req, image_meta, image_data):
         image_meta = self._reserve(req, image_meta)
         image_id = image_meta['id']
 
-        if image_data is not None:
+        if image_data or self._copy_from(req):
             image_meta = self._upload_and_activate(req, image_meta)
         else:
             location = image_meta.get('location')
@@ -594,10 +622,12 @@ def update(self, req, id, image_meta, image_data):
         if image_data is not None and orig_status != 'queued':
             raise HTTPConflict(_("Cannot upload to an unqueued image"))
 
-        # Only allow the Location fields to be modified if the image is
-        # in queued status, which indicates that the user called POST /images
-        # but did not supply either a Location field OR image data
-        if not orig_status == 'queued' and 'location' in image_meta:
+        # Only allow the Location|Copy-From fields to be modified if the
+        # image is in queued status, which indicates that the user called
+        # POST /images but originally supply neither a Location|Copy-From
+        # field NOR image data
+        location = self._external_source(image_meta, req)
+        if not orig_status == 'queued' and location:
             msg = _("Attempted to update Location field for an image "
                     "not in queued status.")
             raise HTTPBadRequest(msg, request=req, content_type="text/plain")

glance/client.py

@@ -130,7 +130,7 @@ def _get_image_size(self, image_data):
                 else:
                     raise
 
-    def add_image(self, image_meta=None, image_data=None):
+    def add_image(self, image_meta=None, image_data=None, features=None):
         """
         Tells Glance about an image's metadata as well
         as optionally the image_data itself
@@ -140,6 +140,7 @@ def add_image(self, image_meta=None, image_data=None):
         :param image_data: Optional string of raw image data
                            or file-like object that can be
                            used to read the image data
+        :param features: Optional map of features
 
         :retval The newly-stored image's metadata.
         """
@@ -155,6 +156,8 @@ def add_image(self, image_meta=None, image_data=None):
         else:
             body = None
 
+        utils.add_features_to_http_headers(features, headers)
+
         res = self.do_request("POST", "/images", body, headers)
         data = json.loads(res.read())
         return data['image']

glance/common/utils.py

@@ -89,6 +89,19 @@ def image_meta_to_http_headers(image_meta):
     return headers
 
 
+def add_features_to_http_headers(features, headers):
+    """
+    Adds additional headers representing glance features to be enabled.
+
+    :param headers: Base set of headers
+    :param features: Map of enabled features
+    """
+    if features:
+        for k, v in features.items():
+            if v is not None:
+                headers[k.lower()] = unicode(v)
+
+
 def get_image_meta_from_headers(response):
     """
     Processes HTTP headers from a supplied response that