Returning functionality of s3 backend to stream remote images

Fixes bug 860872. Also refactoring swift remote image tests to pull location data from registry now that it is unavailable from the main
api.

(Update) Fixed minor feedback for Waldon from Vek.

Change-Id: I159958da8ed4187da2e22392fe341042eedfd056

glance/store/s3.py

@@ -253,7 +253,7 @@ def get(self, location):
         #   raise glance.store.BackendException(msg)
 
         key.BufferSize = self.CHUNKSIZE
-        return (ChunkedFile(key), None)
+        return (ChunkedFile(key), key.size)
 
     def add(self, image_id, image_file, image_size):
         """

glance/tests/functional/test_api.py

@@ -68,7 +68,7 @@ def test_get_head_simple_post(self):
         - Add a previously deleted property.
         """
         self.cleanup()
-        self.start_servers()
+        self.start_servers(**self.__dict__.copy())
 
         # 0. GET /images
         # Verify no public images
@@ -299,7 +299,7 @@ def test_queued_process_flow(self):
         """
 
         self.cleanup()
-        self.start_servers()
+        self.start_servers(**self.__dict__.copy())
 
         # 0. GET /images
         # Verify no public images
@@ -401,7 +401,7 @@ def test_version_variations(self):
         """
 
         self.cleanup()
-        self.start_servers()
+        self.start_servers(**self.__dict__.copy())
 
         versions = {'versions': [{
             "id": "v1.1",
@@ -566,7 +566,7 @@ def test_size_greater_2G_mysql(self):
         """
 
         self.cleanup()
-        self.start_servers()
+        self.start_servers(**self.__dict__.copy())
 
         # 1. POST /images with public image named Image1
         # attribute and a size of 5G. Use the HTTP engine with an
@@ -605,7 +605,7 @@ def test_traceback_not_consumed(self):
         :see https://bugs.launchpad.net/glance/+bug/755912
         """
         self.cleanup()
-        self.start_servers()
+        self.start_servers(**self.__dict__.copy())
 
         # POST /images with binary data, but not setting
         # Content-Type to application/octet-stream, verify a
@@ -630,7 +630,7 @@ def test_filtered_images(self):
         Set up four test images and ensure each query param filter works
         """
         self.cleanup()
-        self.start_servers()
+        self.start_servers(**self.__dict__.copy())
 
         # 0. GET /images
         # Verify no public images
@@ -883,7 +883,7 @@ def test_limited_images(self):
         Ensure marker and limit query params work
         """
         self.cleanup()
-        self.start_servers()
+        self.start_servers(**self.__dict__.copy())
 
         # 0. GET /images
         # Verify no public images
@@ -972,7 +972,7 @@ def test_ordered_images(self):
         Set up three test images and ensure each query param filter works
         """
         self.cleanup()
-        self.start_servers()
+        self.start_servers(**self.__dict__.copy())
 
         # 0. GET /images
         # Verify no public images
@@ -1083,7 +1083,7 @@ def test_duplicate_image_upload(self):
         Upload initial image, then attempt to upload duplicate image
         """
         self.cleanup()
-        self.start_servers()
+        self.start_servers(**self.__dict__.copy())
 
         # 0. GET /images
         # Verify no public images

glance/tests/functional/test_s3.py

@@ -31,15 +31,21 @@
 """
 
 import ConfigParser
+import hashlib
 import json
 import os
 import tempfile
 import unittest
 
+import httplib2
+
 from glance.tests.functional import test_api
 from glance.tests.utils import execute, skip_if_disabled
 
 
+FIVE_KB = 5 * 1024
+
+
 class TestS3(test_api.TestApi):
 
     """Functional tests for the S3 backend"""
@@ -145,3 +151,72 @@ def clear_bucket(self):
         keys = self.bucket.list()
         for key in keys:
             key.delete()
+
+    @skip_if_disabled
+    def test_remote_image(self):
+        """
+        """
+        self.cleanup()
+        self.start_servers(**self.__dict__.copy())
+
+        # 1. POST /images with public image named Image1
+        image_data = "*" * FIVE_KB
+        headers = {'Content-Type': 'application/octet-stream',
+                   'X-Image-Meta-Name': 'Image1',
+                   'X-Image-Meta-Is-Public': 'True'}
+        path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'POST', headers=headers,
+                                         body=image_data)
+        self.assertEqual(response.status, 201)
+        data = json.loads(content)
+        self.assertEqual(data['image']['checksum'],
+                         hashlib.md5(image_data).hexdigest())
+        self.assertEqual(data['image']['size'], FIVE_KB)
+
+        # 2. GET /images/1
+        # Verify all information on image we just added is correct
+        path = "http://%s:%d/v1/images/1" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'GET')
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response['content-length'], str(FIVE_KB))
+        self.assertEqual(content, "*" * FIVE_KB)
+
+        # 3. GET /images/1 from registry in order to find S3 location
+        path = "http://%s:%d/images/1" % ("0.0.0.0", self.registry_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'GET')
+        s3_store_location = json.loads(content)['image']['location']
+
+        # 4. POST /images using location generated by Image1
+        image_data = "*" * FIVE_KB
+        headers = {'Content-Type': 'application/octet-stream',
+                   'X-Image-Meta-Name': 'Image2',
+                   'X-Image-Meta-Is-Public': 'True',
+                   'X-Image-Meta-Location': s3_store_location}
+        path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'POST', headers=headers)
+        self.assertEqual(response.status, 201)
+        self.assertEqual(data['image']['size'], FIVE_KB)
+        self.assertEqual(data['image']['checksum'],
+                         hashlib.md5(image_data).hexdigest())
+
+        # 5. GET /images/2 and make sure it can stream the image
+        path = "http://%s:%d/v1/images/2" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'GET')
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response['content-length'], str(FIVE_KB))
+        self.assertEqual(content, "*" * FIVE_KB)
+
+        # 6. DELETE /images/1 and /images/2
+        path = "http://%s:%d/v1/images/1" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        http.request(path, 'DELETE')
+        path = "http://%s:%d/v1/images/2" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        http.request(path, 'DELETE')
+
+        self.stop_servers()

glance/tests/functional/test_swift.py

@@ -426,8 +426,11 @@ def test_remote_image(self):
         self.assertEqual(hashlib.md5(content).hexdigest(),
                          hashlib.md5("*" * FIVE_MB).hexdigest())
 
-        # use this header as the location for the next image
-        swift_location = response['x-image-meta-location']
+        # GET /images/1 from registry in order to find Swift location
+        path = "http://%s:%d/images/1" % ("0.0.0.0", self.registry_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'GET')
+        swift_location = json.loads(content)['image']['location']
 
         # POST /images with public image named Image1 without uploading data
         image_data = "*" * FIVE_MB

glance/tests/unit/test_s3_store.py

@@ -171,7 +171,7 @@ def test_get(self):
             "s3://user:key@auth_address/glance/2")
         (image_s3, image_size) = self.store.get(loc)
 
-        self.assertEqual(image_size, None)
+        self.assertEqual(image_size, FIVE_KB)
 
         expected_data = "*" * FIVE_KB
         data = ""