Skip to content

Commit

Permalink
Merge pull request fog#1855 from burns/rackspace_object_manifest
Browse files Browse the repository at this point in the history 
[rackspace|storage] allow headers to be specified for object manifest
  • Loading branch information
Kyle Rames committed Jun 5, 2013
2 parents 31c921e + 8900499 commit 16633ed
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 66 deletions.
33 changes: 22 additions & 11 deletions lib/fog/rackspace/requests/storage/put_object_manifest.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,33 @@ class Real

# Create a new manifest object
#
# ==== Parameters
# * container<~String> - Name for container, should be < 256 bytes and must not contain '/'
# * object<~String> - Name for manifest object
# * options<~Hash>:
# * 'segments_container'<~String> - Name of container where segments are stored. +container+ will be used if not given.
# * 'segments_prefix'<~String> - Name prefix used for segmented objects. +object+ will be used if not given.
# @raise [Fog::Storage::Rackspace::NotFound] - HTTP 404
# @raise [Fog::Storage::Rackspace::BadRequest] - HTTP 400
# @raise [Fog::Storage::Rackspace::InternalServerError] - HTTP 500
# Creates an object with a +X-Object-Manifest+ header that specifies the common prefix ("<container>/<prefix>")
# for all uploaded segments. Retrieving the manifest object streams all segments matching this prefix.
# Segments must sort in the order they should be concatenated. Note that any future objects stored in the container
# along with the segments that match the prefix will be included when retrieving the manifest object.
#
# All segments must be stored in the same container, but may be in a different container than the manifest object.
# The default +X-Object-Manifest+ header is set to "+container+/+object+", but may be overridden in +options+
# to specify the prefix and/or the container where segments were stored.
# If overridden, names should be CGI escaped (excluding spaces) if needed (see {Fog::Rackspace.escape}).
#
# @param container [String] Name for container where +object+ will be stored. Should be < 256 bytes and must not contain '/'
# @param object [String] Name for manifest object.
# @param options [Hash] Config headers for +object+.
# @option options [String] 'X-Object-Manifest' ("container/object") "<container>/<prefix>" for segment objects.
#
# @raise [Fog::Storage::Rackspace::NotFound] HTTP 404
# @raise [Fog::Storage::Rackspace::BadRequest] HTTP 400
# @raise [Fog::Storage::Rackspace::InternalServerError] HTTP 500
# @raise [Fog::Storage::Rackspace::ServiceError]
#
# @see http://docs.rackspace.com/files/api/v1/cf-devguide/content/Large_Object_Creation-d1e2019.html
def put_object_manifest(container, object, options = {})
path = "#{Fog::Rackspace.escape(container)}/#{Fog::Rackspace.escape(object)}"
prefix = "#{Fog::Rackspace.escape(options['segments_container'] || container)}/#{Fog::Rackspace.escape(options['segments_prefix'] || object)}"
headers = {'X-Object-Manifest' => path}.merge(options)
request(
:expects => 201,
:headers => {'X-Object-Manifest' => prefix},
:headers => headers,
:method => 'PUT',
:path => path
)
Expand Down
110 changes: 55 additions & 55 deletions tests/rackspace/requests/storage/large_object_tests.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,79 +17,80 @@
Fog::Storage[:rackspace].put_object(@directory.identity, 'fog_large_object/2', ('x' * 2 * 1024 * 1024))
end

tests("#put_object_manifest('foglargeobjecttests', 'fog_large_object')").succeeds do
tests("#put_object('foglargeobjecttests', 'fog_large_object2/1', ('x' * 1 * 1024 * 1024))").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].put_object_manifest(@directory.identity, 'fog_large_object')
Fog::Storage[:rackspace].put_object(@directory.identity, 'fog_large_object2/1', ('x' * 1 * 1024 * 1024))
end

tests("#get_object('foglargeobjecttests', 'fog_large_object').body").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].get_object(@directory.identity, 'fog_large_object').body == ('x' * 6 * 1024 * 1024)
end
tests("using default X-Object-Manifest header") do

tests("returns manifest Etag, computed including manifest file").succeeds do
pending if Fog.mocking?
tests("#put_object_manifest('foglargeobjecttests', 'fog_large_object')").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].put_object_manifest(@directory.identity, 'fog_large_object')
end

etags = []
# When the manifest object name is equal to the prefix for the segments,
# OpenStack treats it as if it's the first segment.
etags << Digest::MD5.hexdigest('') # Etag for manifest object => "d41d8cd98f00b204e9800998ecf8427e"
etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355"
etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51"
expected = Digest::MD5.hexdigest(etags.join) # => "2537ea40a5a8cac247a912906cb62fc2"
actual = Fog::Storage[:rackspace].head_object(@directory.identity, 'fog_large_object').headers['Etag']
actual.gsub('"', '') == expected # actual is returned in quotes "\"2537ea40a5a8cac247a912906cb62fc2"\"
end
tests("#get_object streams all segments matching the default prefix").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].get_object(@directory.identity, 'fog_large_object').body == ('x' * 7 * 1024 * 1024)
end

tests("#delete_object('foglargeobjecttests', 'fog_large_object')").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].delete_object('foglargeobjecttests', 'fog_large_object')
end
tests("#head_object returns Etag that includes manifest object in calculation").succeeds do
pending if Fog.mocking?

etags = []
# When the manifest object name is equal to the prefix, OpenStack treats it as if it's the first segment.
etags << Digest::MD5.hexdigest('') # Etag for manifest object => "d41d8cd98f00b204e9800998ecf8427e"
etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355"
etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51"
etags << Digest::MD5.hexdigest('x' * 1 * 1024 * 1024) # => "b561f87202d04959e37588ee05cf5b10"
expected = Digest::MD5.hexdigest(etags.join) # => "42e92048bd2c8085e7072b0b55fd76ab"
actual = Fog::Storage[:rackspace].head_object(@directory.identity, 'fog_large_object').headers['Etag']
actual.gsub('"', '') == expected # actual is returned in quotes "\"42e92048bd2c8085e7072b0b55fd76abu"\"
end

tests("#put_object_manifest('foglargeobjecttests', 'large_object_manifest', {'segments_prefix' => 'fog_large_object'})").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].put_object_manifest(@directory.identity, 'large_object_manifest', {'segments_prefix' => 'fog_large_object'})
end

tests("#get_object('foglargeobjecttests', 'large_object_manifest').body").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].get_object(@directory.identity, 'large_object_manifest').body == ('x' * 6 * 1024 * 1024)
end
tests("specifying X-Object-Manifest segment prefix") do

tests("returns manifest Etag, computed without manifest file").succeeds do
pending if Fog.mocking?
tests("#put_object_manifest('foglargeobjecttests', 'fog_large_object', {'X-Object-Manifest' => 'foglargeobjecttests/fog_large_object/')").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].put_object_manifest(@directory.identity, 'fog_large_object', {'X-Object-Manifest' => "#{@directory.identity}/fog_large_object/"})
end

etags = []
etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355"
etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51"
expected = Digest::MD5.hexdigest(etags.join) # => "0b348495a774eaa4d4c4bbf770820f84"
actual = Fog::Storage[:rackspace].head_object(@directory.identity, 'large_object_manifest').headers['Etag']
actual.gsub('"', '') == expected # actual is returned in quotes "\"0b348495a774eaa4d4c4bbf770820f84"\"
end
tests("#get_object streams segments only matching the specified prefix").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].get_object(@directory.identity, 'fog_large_object').body == ('x' * 6 * 1024 * 1024)
end

tests("#put_object_manifest('foglargeobjecttests2', 'fog_large_object', {'segments_container' => 'foglargeobjecttests', 'segments_prefix' => 'fog_large_object'})").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].put_object_manifest(@directory2.identity, 'fog_large_object', {'segments_container' => @directory.identity, 'segments_prefix' => 'fog_large_object'})
end
tests("#head_object returns Etag that does not include manifest object in calculation").succeeds do
pending if Fog.mocking?

etags = []
etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355"
etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51"
expected = Digest::MD5.hexdigest(etags.join) # => "0b348495a774eaa4d4c4bbf770820f84"
actual = Fog::Storage[:rackspace].head_object(@directory.identity, 'fog_large_object').headers['Etag']
actual.gsub('"', '') == expected # actual is returned in quotes "\"0b348495a774eaa4d4c4bbf770820f84"\"
end

tests("#get_object('foglargeobjecttests2', 'fog_large_object').body").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].get_object(@directory2.identity, 'fog_large_object').body == ('x' * 6 * 1024 * 1024)
end

tests("returns manifest Etag, computed without manifest file").succeeds do
pending if Fog.mocking?
tests("storing manifest object in a different container than the segments") do

tests("#put_object_manifest('foglargeobjecttests2', 'fog_large_object', {'X-Object-Manifest' => 'foglargeobjecttests/fog_large_object/'})").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].put_object_manifest(@directory2.identity, 'fog_large_object', {'X-Object-Manifest' => "#{@directory.identity}/fog_large_object/"})
end

tests("#get_object('foglargeobjecttests2', 'fog_large_object').body").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].get_object(@directory2.identity, 'fog_large_object').body == ('x' * 6 * 1024 * 1024)
end

etags = []
etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355"
etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51"
expected = Digest::MD5.hexdigest(etags.join) # => "0b348495a774eaa4d4c4bbf770820f84"
actual = Fog::Storage[:rackspace].head_object(@directory2.identity, 'fog_large_object').headers['Etag']
actual.gsub('"', '') == expected # actual is returned in quotes "\"0b348495a774eaa4d4c4bbf770820f84"\"
end

unless Fog.mocking?
['large_object_manifest', 'fog_large_object/1', 'fog_large_object/2'].each do |key|
['fog_large_object', 'fog_large_object/1', 'fog_large_object/2', 'fog_large_object2/1'].each do |key|
@directory.files.new(:key => key).destroy
end
@directory2.files.new(:key => 'fog_large_object').destroy
Expand All @@ -107,5 +108,4 @@
@directory.destroy
@directory2.destroy
end

end

0 comments on commit 16633ed

Please sign in to comment.