Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

change droplet storage to use the Fog gem

Note: this really needs to be refactored.  There is non-trivial duplication
with the AppPackage code, and worse, the legacy staging api implementation is
managing storage now.  It really always was, but it was less in your face
before.

Change-Id: Id72b3f7b513cc05170f6e7e76273ae61501bc764
  • Loading branch information...
commit 1643a289af719471ee7adf2d3dc04f36a905a0b5 1 parent e94149b
@pbozeman pbozeman authored Glenn Oppegard & Patrick Bozeman committed
View
6 config/cloud_controller.yml
@@ -92,3 +92,9 @@ packages:
fog_connection:
provider: Local
local_root: /tmp
+
+droplets:
+ droplet_directory_key: cc-droplets
+ fog_connection:
+ provider: Local
+ local_root: /tmp
View
4 lib/cloud_controller/api/app_bits.rb
@@ -56,7 +56,7 @@ def download(id)
if package_uri.nil?
logger.error "could not find package for #{id}"
- raise AppPackageNotFound.new(id)
+ raise Errors::AppPackageNotFound.new(id)
end
if AppPackage.local?
@@ -73,7 +73,7 @@ def download(id)
def json_param(name)
raw = params[name]
Yajl::Parser.parse(raw)
- rescue Yajl::ParseError => e
+ rescue Yajl::ParseError
raise Errors::AppBitsUploadInvalid.new("invalid :#{name}")
end
View
9 lib/cloud_controller/app_stager.rb
@@ -49,7 +49,7 @@ def stage_app(app)
end
droplet_hash = Digest::SHA1.file(upload_path).hexdigest
- FileUtils.mv(upload_path, droplet_path(app))
+ LegacyStaging.store_droplet(app.guid, upload_path)
app.droplet_hash = droplet_hash
app.save
end
@@ -57,13 +57,8 @@ def stage_app(app)
logger.info "staging for #{app.guid} complete"
end
- def droplet_path(app)
- File.join(droplets_path, "droplet_#{app.guid}")
- end
-
def delete_droplet(app)
- path = droplet_path(app)
- File.delete(path) if File.exists? path
+ LegacyStaging.delete_droplet(app.guid)
end
private
View
11 lib/cloud_controller/config.rb
@@ -113,6 +113,17 @@ class VCAP::CloudController::Config < VCAP::Config
optional(:aws_secret_access_key) => String,
optional(:local_root) => String
}
+ },
+
+ :droplets => {
+ optional(:max_droplet_size) => Integer,
+ optional(:droplet_directory_key) => String,
+ :fog_connection => {
+ :provider => String,
+ optional(:aws_access_key_id) => String,
+ optional(:aws_secret_access_key) => String,
+ optional(:local_root) => String
+ }
}
}
end
View
2  lib/cloud_controller/dea/dea_client.rb
@@ -316,7 +316,7 @@ def start_app_message(app)
:framework => app.framework.name,
:prod => app.production,
:sha1 => app.droplet_hash,
- :executableFile => AppStager.droplet_path(app),
+ :executableFile => "deprecated",
:executableUri => LegacyStaging.droplet_download_uri(app.guid),
:version => app.version,
:services => app.service_bindings.map do |sb|
View
112 lib/cloud_controller/legacy_api/legacy_staging.rb
@@ -1,5 +1,15 @@
# Copyright (c) 2009-2012 VMware, Inc.
+# All the storing and deleting of droplets was pushed into this class as it is
+# a mix of legacy cc and fog related work. This is because this class is
+# responsible for handing out the urls to the droplets, which now gets
+# delegated to fog. There is also substantial overlap in code and functionality
+# with AppPackage.
+#
+# As part of the refactor to use the Fog gem, this ended up being done by
+# dropping Fog in place directly. However, now that there is more going on at
+# the storage layer than FileUtils.mv, this should get refactored.
+
module VCAP::CloudController
class LegacyStaging < LegacyApiBase
include VCAP::CloudController::Errors
@@ -22,10 +32,19 @@ def initialize(id)
class << self
def configure(config)
@config = config
+
+ opts = config[:droplets]
+ @droplet_directory_key = opts[:droplet_directory_key] || "cc-droplets"
+ @connection_config = opts[:fog_connection]
+ @directory = nil
end
def app_uri(id)
- staging_uri("#{APP_PATH}/#{id}")
+ if AppPackage.local?
+ staging_uri("#{APP_PATH}/#{id}")
+ else
+ AppPackage.package_uri(id)
+ end
end
def droplet_upload_uri(id)
@@ -33,7 +52,11 @@ def droplet_upload_uri(id)
end
def droplet_download_uri(id)
- staging_uri("/staged_droplets/#{id}")
+ if local?
+ staging_uri("/staged_droplets/#{id}")
+ else
+ droplet_uri(id)
+ end
end
def with_upload_handle(id)
@@ -49,6 +72,59 @@ def lookup_handle(id)
end
end
+ def store_droplet(guid, path)
+ File.open(path) do |file|
+ droplet_dir.files.create(
+ :key => key_from_guid(guid),
+ :body => file,
+ :public => local?
+ )
+ end
+ end
+
+ def delete_droplet(guid)
+ key = key_from_guid(guid)
+ droplet_dir.files.destroy(key)
+ end
+
+ def droplet_exists?(guid)
+ key = key_from_guid(guid)
+ !droplet_dir.files.head(key).nil?
+ end
+
+ def local?
+ @connection_config[:provider].downcase == "local"
+ end
+
+ # Return droplet uri for path for a given app's guid.
+ #
+ # The url is valid for 1 hour when using aws.
+ # TODO: The expiration should be configurable.
+ def droplet_uri(guid)
+ key = key_from_guid(guid)
+ f = droplet_dir.files.head(key)
+ return nil unless f
+
+ # unfortunately fog doesn't have a unified interface for non-public
+ # urls
+ if local?
+ f.public_url
+ else
+ f.url(Time.now + 3600)
+ end
+ end
+
+ def droplet_local_path(id)
+ raise ArgumentError unless local?
+ key = key_from_guid(id)
+ f = droplet_dir.files.head(key)
+ return nil unless f
+ # Yes, this is bad. But, we really need a handle to the actual path in
+ # order to serve the file using send_file since send_file only takes a
+ # path as an argument
+ f.send(:path)
+ end
+
private
def staging_uri(path)
@@ -93,6 +169,24 @@ def mutex
def logger
@logger ||= Steno.logger("cc.legacy_staging")
end
+
+ def connection
+ opts = @connection_config
+ opts = opts.merge(:endpoint => "") if local?
+ Fog::Storage.new(opts)
+ end
+
+ def droplet_dir
+ @directory ||= connection.directories.create(
+ :key => @droplet_directory_key,
+ :public => false,
+ )
+ end
+
+ def key_from_guid(guid)
+ guid = guid.to_s.downcase
+ File.join(guid[0..1], guid[2..3], guid)
+ end
end
# Handles an app download from a stager
@@ -145,17 +239,25 @@ def upload_droplet(id)
end
def download_droplet(id)
+ raise InvalidRequest unless LegacyStaging.local?
+
app = Models::App.find(:guid => id)
raise AppNotFound.new(id) if app.nil?
- droplet_path = AppStager.droplet_path(app)
- unless droplet_path && File.exists?(droplet_path)
+ droplet_path = LegacyStaging.droplet_local_path(id)
+ logger.debug "id: #{id} droplet_path #{droplet_path}"
+
+ unless droplet_path
+ logger.error "could not find droplet for #{id}"
raise StagingError.new("droplet not found for #{id}")
end
if config[:nginx][:use_nginx]
- return [200, { "X-Accel-Redirect" => "/droplets/" + "droplet_#{id}" }, ""]
+ url = LegacyStaging.droplet_uri(id)
+ logger.debug "nginx redirect #{url}"
+ return [200, { "X-Accel-Redirect" => url }, ""]
else
+ logger.debug "send_file #{droplet_path}"
return send_file droplet_path
end
end
View
27 spec/api/legacy_staging_spec.rb
@@ -32,13 +32,20 @@ module VCAP::CloudController
:provider => "Local",
:local_root => Dir.mktmpdir
}
+ },
+ :droplets => {
+ :fog_connection => {
+ :provider => "Local",
+ :local_root => Dir.mktmpdir
+ }
}
}
end
before do
Fog.unmock!
- LegacyStaging.configure(staging_config)
+ config_override(staging_config)
+ config
end
describe "with_upload_handle" do
@@ -174,10 +181,10 @@ module VCAP::CloudController
context "with a valid droplet" do
xit "should return the droplet" do
- path = AppStager.droplet_path(app_obj)
- File.open(path, "w") do |f|
- f.write("droplet contents")
- end
+ droplet = Tempfile.new(app_obj.guid)
+ droplet.write("droplet contents")
+ droplet.close
+ LegacyStaging.store_droplet(app_obj.guid, droplet.path)
get "/staged_droplets/#{app_obj.guid}"
last_response.status.should == 200
@@ -185,14 +192,14 @@ module VCAP::CloudController
end
it "redirects nginx to serve staged droplet" do
- path = AppStager.droplet_path(app_obj)
- File.open(path, "w") do |f|
- f.write("droplet contents")
- end
+ droplet = Tempfile.new(app_obj.guid)
+ droplet.write("droplet contents")
+ droplet.close
+ LegacyStaging.store_droplet(app_obj.guid, droplet.path)
get "/staged_droplets/#{app_obj.guid}"
last_response.status.should == 200
- last_response.headers["X-Accel-Redirect"].should == "/droplets/droplet_#{app_obj.guid}"
+ last_response.headers["X-Accel-Redirect"].should match("/cc-droplets/.*/#{app_obj.guid}")
end
end
View
26 spec/app_stager_spec.rb
@@ -20,6 +20,14 @@ module VCAP::CloudController
end
it "should return a serialized staging request" do
+ # This should be moved to a helper function
+ guid = app_obj.guid
+ tmpdir = Dir.mktmpdir
+ zipname = File.join(tmpdir, "test.zip")
+ create_zip(zipname, 10, 1024)
+ AppPackage.to_zip(guid, File.new(zipname), [])
+ FileUtils.rm_rf(tmpdir)
+
res = AppStager.send(:staging_request, app_obj)
res.should be_kind_of(Hash)
res[:app_id].should == app_obj.guid
@@ -90,7 +98,7 @@ module VCAP::CloudController
app_obj.needs_staging?.should be_false
app_obj.staged?.should be_true
- File.exists?(AppStager.droplet_path(app_obj)).should be_true
+ LegacyStaging.droplet_exists?(app_obj.guid).should be_true
end
it "should raise a StagingError and propagate the raw description for staging client errors" do
@@ -139,15 +147,21 @@ module VCAP::CloudController
let(:app_obj) { Models::App.make }
it "should do nothing if the droplet does not exist" do
- File.should_receive(:exists?).and_return(false)
- File.should_not_receive(:delete)
+ guid = Sham.guid
+ LegacyStaging.droplet_exists?(guid).should == false
AppStager.delete_droplet(app_obj)
+ LegacyStaging.droplet_exists?(guid).should == false
end
- it "should delete the droploet if it exists" do
- File.should_receive(:exists?).and_return(true)
- File.should_receive(:delete).with(AppStager.droplet_path(app_obj))
+ it "should delete the droplet if it exists" do
+ droplet = Tempfile.new(app_obj.guid)
+ droplet.write("droplet contents")
+ droplet.close
+ LegacyStaging.store_droplet(app_obj.guid, droplet.path)
+
+ LegacyStaging.droplet_exists?(app_obj.guid).should == true
AppStager.delete_droplet(app_obj)
+ LegacyStaging.droplet_exists?(app_obj.guid).should == false
end
end
end
View
8 spec/spec_helper.rb
@@ -100,6 +100,14 @@ def config
:aws_access_key_id => "fake_aws_key_id",
:aws_secret_access_key => "fake_secret_access_key",
}
+ },
+ :droplets => {
+ :droplet_directory_key => "cc-droplets",
+ :fog_connection => {
+ :provider => "AWS",
+ :aws_access_key_id => "fake_aws_key_id",
+ :aws_secret_access_key => "fake_secret_access_key",
+ }
}
)
Please sign in to comment.
Something went wrong with that request. Please try again.