Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge "Add support for OpenStack Swift to blobstore client"

  • Loading branch information...
commit c53ab340e86ec047022cf864502bb5648d90c646 2 parents 67ba02a + 7b007d7
@frodenas frodenas authored Gerrit Code Review committed
View
17 blobstore_client/Gemfile.lock
@@ -4,6 +4,7 @@ PATH
blobstore_client (0.4.0)
aws-s3 (~> 0.6.2)
bosh_common (~> 0.5)
+ fog (~> 1.5.0)
httpclient (>= 2.2)
multi_json (~> 1.1.0)
ruby-atmos-pure (~> 1.0.5)
@@ -23,6 +24,18 @@ GEM
builder (>= 2.1.2)
columnize (0.3.3)
diff-lcs (1.1.3)
+ excon (0.16.1)
+ fog (1.5.0)
+ builder
+ excon (~> 0.14)
+ formatador (~> 0.2.0)
+ mime-types
+ multi_json (~> 1.0)
+ net-scp (~> 1.0.4)
+ net-ssh (>= 2.1.3)
+ nokogiri (~> 1.5.0)
+ ruby-hmac
+ formatador (0.2.3)
httpclient (2.2.4)
linecache (0.46)
rbx-require-relative (> 0.0.4)
@@ -31,6 +44,10 @@ GEM
log4r (1.1.10)
mime-types (1.17.2)
multi_json (1.1.0)
+ net-scp (1.0.4)
+ net-ssh (>= 1.99.1)
+ net-ssh (2.5.2)
+ nokogiri (1.5.5)
rake (0.9.2.2)
rbx-require-relative (0.0.5)
rcov (0.9.9)
View
1  blobstore_client/blobstore_client.gemspec
@@ -19,6 +19,7 @@ Gem::Specification.new do |s|
s.executables = %w(blobstore_client_console)
s.add_dependency "aws-s3", "~> 0.6.2"
+ s.add_dependency "fog", "~>1.5.0"
s.add_dependency "httpclient", ">=2.2"
s.add_dependency "multi_json", "~> 1.1.0"
s.add_dependency "ruby-atmos-pure", "~> 1.0.5"
View
2  blobstore_client/lib/blobstore_client.rb
@@ -10,6 +10,7 @@ module Bosh; module Blobstore; end; end
require "blobstore_client/base"
require "blobstore_client/simple_blobstore_client"
require "blobstore_client/s3_blobstore_client"
+require "blobstore_client/swift_blobstore_client"
require "blobstore_client/local_client"
require "blobstore_client/atmos_blobstore_client"
@@ -20,6 +21,7 @@ class Client
PROVIDER_MAP = {
"simple" => SimpleBlobstoreClient,
"s3" => S3BlobstoreClient,
+ "swift" => SwiftBlobstoreClient,
"atmos" => AtmosBlobstoreClient,
"local" => LocalClient
}
View
160 blobstore_client/lib/blobstore_client/swift_blobstore_client.rb
@@ -0,0 +1,160 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require "base64"
+require "fog"
+require "multi_json"
+require "uri"
+require "uuidtools"
+
+module Bosh
+ module Blobstore
+
+ class SwiftBlobstoreClient < BaseClient
+
+ # Blobstore client for Swift
+ # @param [Hash] options Swift BlobStore options
+ # @option options [Symbol] container_name
+ # @option options [Symbol] swift_provider
+ def initialize(options)
+ super(options)
+ @http_client = HTTPClient.new
+ end
+
+ def container
+ return @container if @container
+
+ validate_options(@options)
+
+ swift_provider = @options[:swift_provider]
+ swift_options = {:provider => swift_provider}
+ swift_options.merge!(@options[swift_provider.to_sym])
+ swift = Fog::Storage.new(swift_options)
+
+ container_name = @options[:container_name]
+ @container = swift.directories.get(container_name)
+ if @container.nil?
+ raise NotFound, "Swift container '#{container_name}' not found"
+ end
+ @container
+ end
+
+ def create_file(file)
+ object_id = generate_object_id
+ object = container.files.create(:key => object_id,
+ :body => file,
+ :public => true)
+ encode_object_id(object_id, object.public_url)
+ rescue Exception => e
+ raise BlobstoreError, "Failed to create object: #{e.message}"
+ end
+
+ def get_file(object_id, file)
+ object_info = decode_object_id(object_id)
+ if object_info["purl"]
+ response = @http_client.get(object_info["purl"]) do |block|
+ file.write(block)
+ end
+ if response.status != 200
+ raise BlobstoreError, "Could not fetch object, %s/%s" %
+ [response.status, response.content]
+ end
+ else
+ object = container.files.get(object_info["oid"]) do |block|
+ file.write(block)
+ end
+ if object.nil?
+ raise NotFound, "Swift object '#{object_id}' not found"
+ end
+ end
+ rescue Exception => e
+ raise BlobstoreError,
+ "Failed to find object '#{object_id}': #{e.message}"
+ end
+
+ def delete(object_id)
+ object_info = decode_object_id(object_id)
+ object = container.files.get(object_info["oid"])
+ if object.nil?
+ raise NotFound, "Swift object '#{object_id}' not found"
+ else
+ object.destroy
+ end
+ rescue Exception => e
+ raise BlobstoreError,
+ "Failed to delete object '#{object_id}': #{e.message}"
+ end
+
+ private
+
+ def generate_object_id
+ UUIDTools::UUID.random_create.to_s
+ end
+
+ def encode_object_id(object_id, public_url = nil)
+ json = MultiJson.encode({:oid => object_id, :purl => public_url})
+ URI::escape(Base64.encode64(json))
+ end
+
+ def decode_object_id(object_id)
+ begin
+ object_info = MultiJson.decode(Base64.decode64(URI::unescape(object_id)))
+ rescue MultiJson::DecodeError => e
+ raise BlobstoreError, "Failed to parse object_id: #{e.message}"
+ end
+
+ if !object_info.kind_of?(Hash) || object_info["oid"].nil?
+ raise BlobstoreError, "Invalid object_id: #{object_info.inspect}"
+ end
+ object_info
+ end
+
+ def validate_options(options)
+ unless options.is_a?(Hash)
+ raise "Invalid options format, Hash expected, #{options.class} given"
+ end
+ unless options.has_key?(:container_name)
+ raise "Swift container name is missing"
+ end
+ unless options.has_key?(:swift_provider)
+ raise "Swift provider is missing"
+ end
+ case options[:swift_provider]
+ when "hp"
+ unless options.has_key?(:hp)
+ raise "HP options are missing"
+ end
+ unless options[:hp].is_a?(Hash)
+ raise "Invalid HP options, Hash expected,
+ #{options[:hp].class} given"
+ end
+ unless options[:hp].has_key?(:hp_account_id)
+ raise "HP account ID is missing"
+ end
+ unless options[:hp].has_key?(:hp_secret_key)
+ raise "HP secret key is missing"
+ end
+ unless options[:hp].has_key?(:hp_tenant_id)
+ raise "HP tenant ID is missing"
+ end
+ when "rackspace"
+ unless options.has_key?(:rackspace)
+ raise "Rackspace options are missing"
+ end
+ unless options[:rackspace].is_a?(Hash)
+ raise "Invalid Rackspace options, Hash expected,
+ #{options[:rackspace].class} given"
+ end
+ unless options[:rackspace].has_key?(:rackspace_username)
+ raise "Rackspace username is missing"
+ end
+ unless options[:rackspace].has_key?(:rackspace_api_key)
+ raise "Rackspace API key is missing"
+ end
+ else
+ raise "Swift provider #{options[:swift_provider]} not supported"
+ end
+ end
+
+ end
+ end
+end
View
5 blobstore_client/spec/unit/blobstore_client_spec.rb
@@ -30,6 +30,11 @@
bs.should be_instance_of Bosh::Blobstore::S3BlobstoreClient
end
+ it "should have an swift provider" do
+ bs = Bosh::Blobstore::Client.create('swift', {})
+ bs.should be_instance_of Bosh::Blobstore::SwiftBlobstoreClient
+ end
+
it "should raise an exception on an unknown client" do
lambda {
bs = Bosh::Blobstore::Client.create('foobar', {})
View
315 blobstore_client/spec/unit/swift_blobstore_client_spec.rb
@@ -0,0 +1,315 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Bosh::Blobstore::SwiftBlobstoreClient do
+
+ def swift_options(container_name, swift_provider, credentials)
+ if credentials
+ options = {
+ "rackspace" => {
+ "rackspace_username" => "username",
+ "rackspace_api_key" => "api_key"
+ },
+ "hp" => {
+ "hp_account_id" => "account_id",
+ "hp_secret_key" => "secret_key",
+ "hp_tenant_id" => "tenant_id"
+ }
+ }
+ else
+ options = {}
+ end
+ options["container_name"] = container_name if container_name
+ options["swift_provider"] = swift_provider if swift_provider
+ options
+ end
+
+ def swift_blobstore(options)
+ Bosh::Blobstore::SwiftBlobstoreClient.new(options)
+ end
+
+ before(:each) do
+ @swift = mock("swift")
+ Fog::Storage.stub!(:new).and_return(@swift)
+ @http_client = mock("http-client")
+ HTTPClient.stub!(:new).and_return(@http_client)
+ end
+
+ describe "on HP Cloud Storage" do
+
+ describe "with credentials" do
+
+ before(:each) do
+ @client = swift_blobstore(swift_options("test-container",
+ "hp",
+ true))
+ end
+
+ it "should create an object" do
+ data = "some content"
+ directories = double("directories")
+ container = double("container")
+ files = double("files")
+ object = double("object")
+
+ @client.should_receive(:generate_object_id).and_return("object_id")
+ @swift.stub(:directories).and_return(directories)
+ directories.should_receive(:get).with("test-container") \
+ .and_return(container)
+ container.should_receive(:files).and_return(files)
+ files.should_receive(:create).with { |opt|
+ opt[:key].should eql "object_id"
+ #opt[:body].should eql data
+ opt[:public].should eql true
+ }.and_return(object)
+ object.should_receive(:public_url).and_return("public-url")
+
+ object_id = @client.create(data)
+ object_info = MultiJson.decode(Base64.decode64(
+ URI::unescape(object_id)))
+ object_info["oid"].should eql("object_id")
+ object_info["purl"].should eql("public-url")
+ end
+
+ it "should fetch an object without a public url" do
+ data = "some content"
+ directories = double("directories")
+ container = double("container")
+ files = double("files")
+ object = double("object")
+
+ @swift.stub(:directories).and_return(directories)
+ directories.should_receive(:get).with("test-container") \
+ .and_return(container)
+ container.should_receive(:files).and_return(files)
+ files.should_receive(:get).with("object_id").and_yield(data) \
+ .and_return(object)
+
+ oid = URI::escape(Base64.encode64(MultiJson.encode(
+ {:oid => "object_id"})))
+ @client.get(oid).should eql(data)
+ end
+
+ it "should fetch an object with a public url" do
+ data = "some content"
+ response = mock("response")
+
+ @http_client.should_receive(:get).with("public-url") \
+ .and_yield(data).and_return(response)
+ response.stub!(:status).and_return(200)
+
+ oid = URI::escape(Base64.encode64(MultiJson.encode(
+ {:oid => "object_id",
+ :purl => "public-url"})))
+ @client.get(oid).should eql(data)
+ end
+
+ it "should delete an object" do
+ directories = double("directories")
+ container = double("container")
+ files = double("files")
+ object = double("object")
+
+ @swift.stub(:directories).and_return(directories)
+ directories.should_receive(:get).with("test-container") \
+ .and_return(container)
+ container.should_receive(:files).and_return(files)
+ files.should_receive(:get).with("object_id").and_return(object)
+ object.should_receive(:destroy)
+
+ oid = URI::escape(Base64.encode64(MultiJson.encode(
+ {:oid => "object_id"})))
+ @client.delete(oid)
+ end
+
+ end
+
+ describe "without credentials" do
+
+ before(:each) do
+ @client = swift_blobstore(swift_options("test-container",
+ "hp",
+ false))
+ end
+
+ it "should refuse to create an object" do
+ data = "some content"
+
+ lambda {
+ object_id = @client.create(data)
+ }.should raise_error(Bosh::Blobstore::BlobstoreError)
+ end
+
+ it "should refuse to fetch an object without a public url" do
+ oid = URI::escape(Base64.encode64(MultiJson.encode(
+ {:oid => "object_id"})))
+ lambda {
+ @client.get(oid)
+ }.should raise_error(Bosh::Blobstore::BlobstoreError)
+ end
+
+ it "should fetch an object with a public url" do
+ data = "some content"
+ response = mock("response")
+
+ @http_client.should_receive(:get).with("public-url") \
+ .and_yield(data).and_return(response)
+ response.stub!(:status).and_return(200)
+
+ oid = URI::escape(Base64.encode64(MultiJson.encode(
+ {:oid => "object_id",
+ :purl => "public-url"})))
+ @client.get(oid).should eql(data)
+ end
+
+ it "should refuse to delete an object" do
+ oid = URI::escape(Base64.encode64(MultiJson.encode(
+ {:oid => "object_id"})))
+ lambda {
+ @client.delete(oid)
+ }.should raise_error(Bosh::Blobstore::BlobstoreError)
+ end
+
+ end
+
+ end
+
+ describe "on Rackspace Cloud Files" do
+
+ describe "with credentials" do
+
+ before(:each) do
+ @client = swift_blobstore(swift_options("test-container",
+ "rackspace",
+ true))
+ end
+
+ it "should create an object" do
+ data = "some content"
+ directories = double("directories")
+ container = double("container")
+ files = double("files")
+ object = double("object")
+
+ @client.should_receive(:generate_object_id).and_return("object_id")
+ @swift.stub(:directories).and_return(directories)
+ directories.should_receive(:get).with("test-container") \
+ .and_return(container)
+ container.should_receive(:files).and_return(files)
+ files.should_receive(:create).with { |opt|
+ opt[:key].should eql "object_id"
+ #opt[:body].should eql data
+ opt[:public].should eql true
+ }.and_return(object)
+ object.should_receive(:public_url).and_return("public-url")
+
+ object_id = @client.create(data)
+ object_info = MultiJson.decode(Base64.decode64(
+ URI::unescape(object_id)))
+ object_info["oid"].should eql("object_id")
+ object_info["purl"].should eql("public-url")
+ end
+
+ it "should fetch an object without a public url" do
+ data = "some content"
+ directories = double("directories")
+ container = double("container")
+ files = double("files")
+ object = double("object")
+
+ @swift.stub(:directories).and_return(directories)
+ directories.should_receive(:get).with("test-container") \
+ .and_return(container)
+ container.should_receive(:files).and_return(files)
+ files.should_receive(:get).with("object_id").and_yield(data) \
+ .and_return(object)
+
+ oid = URI::escape(Base64.encode64(MultiJson.encode(
+ {:oid => "object_id"})))
+ @client.get(oid).should eql(data)
+ end
+
+ it "should fetch an object with a public url" do
+ data = "some content"
+ response = mock("response")
+
+ @http_client.should_receive(:get).with("public-url") \
+ .and_yield(data).and_return(response)
+ response.stub!(:status).and_return(200)
+
+ oid = URI::escape(Base64.encode64(MultiJson.encode(
+ {:oid => "object_id",
+ :purl => "public-url"})))
+ @client.get(oid).should eql(data)
+ end
+
+ it "should delete an object" do
+ directories = double("directories")
+ container = double("container")
+ files = double("files")
+ object = double("object")
+
+ @swift.stub(:directories).and_return(directories)
+ directories.should_receive(:get).with("test-container") \
+ .and_return(container)
+ container.should_receive(:files).and_return(files)
+ files.should_receive(:get).with("object_id").and_return(object)
+ object.should_receive(:destroy)
+
+ oid = URI::escape(Base64.encode64(MultiJson.encode(
+ {:oid => "object_id"})))
+ @client.delete(oid)
+ end
+
+ end
+
+ describe "without credentials" do
+
+ before(:each) do
+ @client = swift_blobstore(swift_options("test-container",
+ "rackspace",
+ false))
+ end
+
+ it "should refuse to create an object" do
+ data = "some content"
+
+ lambda {
+ object_id = @client.create(data)
+ }.should raise_error(Bosh::Blobstore::BlobstoreError)
+ end
+
+ it "should refuse to fetch an object without a public url" do
+ oid = URI::escape(Base64.encode64(MultiJson.encode(
+ {:oid => "object_id"})))
+ lambda {
+ @client.get(oid)
+ }.should raise_error(Bosh::Blobstore::BlobstoreError)
+ end
+
+ it "should fetch an object with a public url" do
+ data = "some content"
+ response = mock("response")
+
+ @http_client.should_receive(:get).with("public-url") \
+ .and_yield(data).and_return(response)
+ response.stub!(:status).and_return(200)
+
+ oid = URI::escape(Base64.encode64(MultiJson.encode(
+ {:oid => "object_id",
+ :purl => "public-url"})))
+ @client.get(oid).should eql(data)
+ end
+
+ it "should refuse to delete an object" do
+ oid = URI::escape(Base64.encode64(MultiJson.encode(
+ {:oid => "object_id"})))
+ lambda {
+ @client.delete(oid)
+ }.should raise_error(Bosh::Blobstore::BlobstoreError)
+ end
+
+ end
+
+ end
+
+end
View
BIN  blobstore_client/vendor/cache/excon-0.16.1.gem
Binary file not shown
View
BIN  blobstore_client/vendor/cache/fog-1.5.0.gem
Binary file not shown
View
BIN  blobstore_client/vendor/cache/formatador-0.2.3.gem
Binary file not shown
View
BIN  blobstore_client/vendor/cache/net-scp-1.0.4.gem
Binary file not shown
View
BIN  blobstore_client/vendor/cache/net-ssh-2.5.2.gem
Binary file not shown
View
BIN  blobstore_client/vendor/cache/nokogiri-1.5.5.gem
Binary file not shown
Please sign in to comment.
Something went wrong with that request. Please try again.