Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

change AppPackage to use Fog Gem

The Fog gem provides all the indirection for local storage (over nfs) or s3
access.

Change-Id: I4877f04fa065c641b1b1d6021c3d1e6170ba2188
  • Loading branch information...
commit e94149bf0798334107c70de6d95ad6ac1945fd85 1 parent c0bbedd
@pbozeman pbozeman authored Glenn Oppegard & Patrick Bozeman committed
View
6 config/cloud_controller.yml
@@ -86,3 +86,9 @@ resource_pool:
fog_connection:
provider: Local
local_root: /tmp
+
+packages:
+ resource_directory_key: cc-packages
+ fog_connection:
+ provider: Local
+ local_root: /tmp
View
21 lib/cloud_controller/api/app_bits.rb
@@ -49,17 +49,24 @@ def upload(id)
end
def download(id)
- app = find_id_and_validate_access(:read, id)
+ find_id_and_validate_access(:read, id)
- package_path = AppPackage.package_path(id)
- unless File.exist?(package_path)
- raise Errors::AppPackageNotFound.new(id)
+ package_uri = AppPackage.package_uri(id)
+ logger.debug "id: #{id} package_uri: #{package_uri}"
+
+ if package_uri.nil?
+ logger.error "could not find package for #{id}"
+ raise AppPackageNotFound.new(id)
end
- if config[:nginx][:use_nginx]
- return [200, { "X-Accel-Redirect" => "/droplets/" + "app_#{id}" }, ""]
+ if AppPackage.local?
+ if config[:nginx][:use_nginx]
+ return [200, { "X-Accel-Redirect" => "/droplets#{package_uri}" }, ""]
+ else
+ return send_file package_path, :filename => File.basename("#{path}.zip")
+ end
else
- return send_file package_path, :filename => File.basename("#{path}.zip")
+ return [HTTP::FOUND, {"Location" => package_uri}, nil]
end
end
View
87 lib/cloud_controller/app_package.rb
@@ -10,8 +10,11 @@ module AppPackage
class << self
# Configure the AppPackage
def configure(config = {})
- @config = config.dup
- @max_droplet_size = @config[:max_droplet_size] || 512 * 1024 * 1024
+ opts = config[:packages]
+ @app_package_directory_key = opts[:app_package_directory_key] || "cc-app-packages"
+ @connection_config = opts[:fog_connection]
+ @max_droplet_size = opts[:max_droplet_size] || 512 * 1024 * 1024
+ @directory = nil
end
# Collects the necessary files and returns the sha1 of the resulting
@@ -27,7 +30,15 @@ def to_zip(guid, uploaded_file, resources)
# Do the sha1 before the mv, because the mv might be to a slower store
sha1 = Digest::SHA1.file(repacked_path).hexdigest
- FileUtils.mv(repacked_path, package_path(guid))
+
+ File.open(repacked_path) do |file|
+ package_dir.files.create(
+ :key => key_from_guid(guid),
+ :body => file,
+ :public => local?
+ )
+ end
+
sha1
ensure
FileUtils.rm_rf(tmpdir) if tmpdir
@@ -36,28 +47,42 @@ def to_zip(guid, uploaded_file, resources)
end
def delete_package(guid)
- path = package_path(guid)
- File.delete(path) if File.exists? path
+ key = key_from_guid(guid)
+ package_dir.files.destroy(key)
+ end
+
+ def package_exists?(guid)
+ key = key_from_guid(guid)
+ !package_dir.files.head(key).nil?
end
- # Return the package directory.
+ # Return app uri for path for a given app's guid.
#
- # Makes the directory on first use.
- def package_dir
- if @config[:directories] && @config[:directories][:droplets]
- FileUtils.mkdir_p(@config[:directories][:droplets])
+ # The url is valid for 1 hour when using aws.
+ # TODO: The expiration should be configurable.
+ def package_uri(guid)
+ key = key_from_guid(guid)
+ f = package_dir.files.head(key)
+ return nil unless f
+
+ # unfortunately fog doesn't have a unified interface for non-public
+ # urls
+ if local?
+ f.public_url
else
- # TODO: remove this tmpdir. It is for use when running under vcap
- # for development
- @config[:directories] ||= {}
- @config[:directories][:droplets] = Dir.mktmpdir
+ f.url(Time.now + 3600)
end
- @config[:directories][:droplets]
end
- # Return app package path for a given app's guid.
- def package_path(guid)
- File.join(package_dir, "app_#{guid}")
+ def package_local_path(guid)
+ raise ArgumentError unless local?
+ key = key_from_guid(guid)
+ f = package_dir.files.head(key)
+ return nil unless f
+ # Yes, this is bad. But, we really need a handle to the actual path in
+ # order to serve the file using send_file since send_file only takes a
+ # path as an argument
+ f.send(:path)
end
# Unzip the uploaded file
@@ -159,7 +184,7 @@ def synchronize_pool_with(working_dir, resource_descriptors)
end
rescue => e
logger.error "failed synchronizing resource pool with '#{working_dir}' #{e}"
- raise Errors::AppPackageInvalid.new("failed synchronizing resource pool")
+ raise Errors::AppPackageInvalid.new("failed synchronizing resource pool #{e}")
end
# Repacks a directory into a compressed file.
@@ -185,7 +210,29 @@ def logger
@logger ||= Steno.logger("cc.ap")
end
- attr_accessor :max_droplet_size, :droplets_dir
+ def connection
+ opts = @connection_config
+ opts = opts.merge(:endpoint => "") if local?
+ Fog::Storage.new(opts)
+ end
+
+ def package_dir
+ @directory ||= connection.directories.create(
+ :key => @app_package_directory_key,
+ :public => false,
+ )
+ end
+
+ def key_from_guid(guid)
+ guid = guid.to_s.downcase
+ File.join(guid[0..1], guid[2..3], guid)
+ end
+
+ def local?
+ @connection_config[:provider].downcase == "local"
+ end
+
+ attr_accessor :max_droplet_size
end
end
end
View
11 lib/cloud_controller/config.rb
@@ -102,6 +102,17 @@ class VCAP::CloudController::Config < VCAP::Config
optional(:aws_secret_access_key) => String,
optional(:local_root) => String
}
+ },
+
+ :packages => {
+ optional(:max_droplet_size) => Integer,
+ optional(:app_package_directory_key) => String,
+ :fog_connection => {
+ :provider => String,
+ optional(:aws_access_key_id) => String,
+ optional(:aws_secret_access_key) => String,
+ optional(:local_root) => String
+ }
}
}
end
View
11 lib/cloud_controller/legacy_api/legacy_staging.rb
@@ -97,20 +97,25 @@ def logger
# Handles an app download from a stager
def download_app(id)
+ raise InvalidRequest unless AppPackage.local?
+
app = Models::App.find(:guid => id)
raise AppNotFound.new(id) if app.nil?
- package_path = AppPackage.package_path(id)
+ package_path = AppPackage.package_local_path(id)
logger.debug "id: #{id} package_path: #{package_path}"
- unless File.exist?(package_path)
+ unless package_path
logger.error "could not find package for #{id}"
raise AppPackageNotFound.new(id)
end
if config[:nginx][:use_nginx]
- return [200, { "X-Accel-Redirect" => "/droplets/" + "app_#{id}" }, ""]
+ url = AppPackage.package_uri(id)
+ logger.debug "nginx redirect #{url}"
+ return [200, { "X-Accel-Redirect" => url }, ""]
else
+ logger.debug "send_file #{patchage_path}"
return send_file package_path
end
end
View
12 lib/cloud_controller/resource_pool.rb
@@ -45,11 +45,13 @@ def add_path(path)
key = key_from_sha1(sha1)
return if resource_dir.files.head(sha1)
- resource_dir.files.create(
- :key => key,
- :body => File.open(path),
- :public => false,
- )
+ File.open(path) do |file|
+ resource_dir.files.create(
+ :key => key,
+ :body => file,
+ :public => false,
+ )
+ end
end
def resource_sizes(resources)
View
21 spec/api/app_bits_spec.rb
@@ -105,17 +105,12 @@ module VCAP::CloudController
let(:developer2) { make_developer_for_space(app_obj_without_pkg.space) }
before do
- AppPackage.configure(config_override({
- :directories => { :droplets => tmpdir }
- }))
-
- pkg_path = AppPackage.package_path(app_obj.guid)
- File.open(pkg_path, "w") do |f|
- f.write("A")
- end
- end
-
- after do
+ config
+ guid = app_obj.guid
+ tmpdir = Dir.mktmpdir
+ zipname = File.join(tmpdir, "test.zip")
+ create_zip(zipname, 10, 1024)
+ AppPackage.to_zip(guid, File.new(zipname), [])
FileUtils.rm_rf(tmpdir)
end
@@ -124,10 +119,10 @@ module VCAP::CloudController
get "/v2/apps/#{app_obj_without_pkg.guid}/download", {}, headers_for(developer2)
last_response.status.should == 404
end
- it "should return 200 for valid packages" do
+ it "should return 302 for valid packages" do
get "/v2/apps/#{app_obj.guid}/download", {}, headers_for(developer)
puts last_response.body
- last_response.status.should == 200
+ last_response.status.should == 302
end
it "should return 404 for non-existent apps" do
get "/v2/apps/abcd/download", {}, headers_for(developer)
View
27 spec/api/legacy_staging_spec.rb
@@ -20,11 +20,24 @@ module VCAP::CloudController
:user => staging_user,
:password => staging_password
}
+ },
+ :resource_pool => {
+ :fog_connection => {
+ :provider => "Local",
+ :local_root => Dir.mktmpdir
+ }
+ },
+ :packages => {
+ :fog_connection => {
+ :provider => "Local",
+ :local_root => Dir.mktmpdir
+ }
}
}
end
before do
+ Fog.unmock!
LegacyStaging.configure(staging_config)
end
@@ -80,20 +93,22 @@ module VCAP::CloudController
before do
config_override(staging_config)
authorize staging_user, staging_password
- AppPackage.configure
- pkg_path = AppPackage.package_path(app_obj.guid)
- File.open(pkg_path, "w") do |f|
- f.write("A")
- end
end
it "should succeed for valid packages" do
+ guid = app_obj.guid
+ tmpdir = Dir.mktmpdir
+ zipname = File.join(tmpdir, "test.zip")
+ create_zip(zipname, 10, 1024)
+ AppPackage.to_zip(guid, File.new(zipname), [])
+ FileUtils.rm_rf(tmpdir)
+
get "/staging/apps/#{app_obj.guid}"
last_response.status.should == 200
end
it "should return an error for non-existent apps" do
- get "/staging/apps/abcd"
+ get "/staging/apps/#{Sham.guid}"
last_response.status.should == 404
end
View
94 spec/app_package_spec.rb
@@ -7,15 +7,22 @@ module VCAP::CloudController
include_context "resource pool"
let(:tmpdir) { Dir.mktmpdir }
- let(:droplets_dir) { Dir.mktmpdir }
before do
- AppPackage.configure(:directories => { :droplets => droplets_dir })
+ AppPackage.configure(
+ :packages => {
+ :fog_connection => {
+ :provider => "AWS",
+ :aws_access_key_id => "fake_aws_key_id",
+ :aws_secret_access_key => "fake_secret_access_key",
+ }
+ }
+ )
+ Fog.mock!
end
after do
FileUtils.rm_rf(tmpdir)
- FileUtils.rm_rf(droplets_dir)
end
describe 'unzipped_size' do
@@ -152,11 +159,11 @@ module VCAP::CloudController
describe "to_zip" do
it "should move the app package to the droplets directory" do
- guid = "abc"
+ guid = Sham.guid
zipname = File.join(tmpdir, "test.zip")
create_zip(zipname, 10, 1024)
AppPackage.to_zip(guid, File.new(zipname), [])
- File.exist?(AppPackage.package_path(guid)).should == true
+ AppPackage.package_exists?(guid).should == true
end
end
@@ -166,51 +173,62 @@ module VCAP::CloudController
end
it "should do nothing if the app package does not exist" do
- File.should_receive(:exists?).and_return(false)
- File.should_not_receive(:delete)
- AppPackage.delete_package("some_guid")
+ guid = Sham.guid
+
+ # It is hard to test this via Fog, but lets at least make sure that it
+ # doesn't throw an exception
+ AppPackage.package_exists?(guid).should == false
+ AppPackage.delete_package(guid)
+ AppPackage.package_exists?(guid).should == false
end
it "should delete the droplet if it exists" do
- File.should_receive(:exists?).and_return(true)
- File.should_receive(:delete).with(AppPackage.package_path("some_guid"))
- AppPackage.delete_package("some_guid")
- end
- end
- end
+ guid = Sham.guid
- describe "#package_dir" do
- subject { AppPackage.package_dir }
- let(:config) { { :directories => { :droplets => "/from_config" } } }
- before { AppPackage.configure(config) }
+ AppPackage.package_exists?(guid).should == false
+ zipname = File.join(tmpdir, "test.zip")
+ create_zip(zipname, 10, 1024)
+ AppPackage.to_zip(guid, File.new(zipname), [])
+ AppPackage.package_exists?(guid).should == true
- context "when droplets directory was defined in configuration" do
- it "creates defined droplets directory" do
- FileUtils.should_receive(:mkdir_p).with("/from_config")
- subject
+ AppPackage.delete_package(guid)
+ AppPackage.package_exists?(guid).should == false
end
end
- shared_examples "creating and setting droplets directory" do
- it "creates temporary droplets directory" do
- Dir.should_receive(:mktmpdir)
- subject
+ describe "package_uri" do
+ before do
+ @guid = Sham.guid
+
+ AppPackage.configure(
+ :packages => {
+ :fog_connection => {
+ :provider => "AWS",
+ :aws_access_key_id => "fake_aws_key_id",
+ :aws_secret_access_key => "fake_secret_access_key",
+ }
+ }
+ )
+ Fog.mock!
+
+ tmpdir = Dir.mktmpdir
+ AppPackage.package_exists?(@guid).should == false
+ zipname = File.join(tmpdir, "test.zip")
+ create_zip(zipname, 10, 1024)
+ AppPackage.to_zip(@guid, File.new(zipname), [])
+ AppPackage.package_exists?(@guid).should == true
+ FileUtils.rm_rf(tmpdir)
end
- it "sets temporary droplets directory in configuration" do
- Dir.should_receive(:mktmpdir) { "/temporary_dir" }
- subject.should eq "/temporary_dir"
+ it "should return a URL for a valid guid" do
+ uri = AppPackage.package_uri(@guid)
+ uri.should match(/https:\/\/.*s3.amazonaws.com\/.*/)
end
- end
- context "when directories were not defined in configuration" do
- let(:config) { {} }
- include_examples "creating and setting droplets directory"
- end
-
- context "when droplets directory was not defined in configuration" do
- let(:config) { { :directories => { } } }
- include_examples "creating and setting droplets directory"
+ it "should return nil for an invalid guid" do
+ uri = AppPackage.package_uri(Sham.guid)
+ uri.should be_nil
+ end
end
end
end
View
16 spec/spec_helper.rb
@@ -92,10 +92,24 @@ def config
:aws_access_key_id => "fake_aws_key_id",
:aws_secret_access_key => "fake_secret_access_key",
}
+ },
+ :packages => {
+ :app_package_directory_key => "cc-packages",
+ :fog_connection => {
+ :provider => "AWS",
+ :aws_access_key_id => "fake_aws_key_id",
+ :aws_secret_access_key => "fake_secret_access_key",
+ }
}
)
- Fog.mock!
+
c = c.merge(@config_override || {})
+
+ unless (c[:resource_pool][:fog_connection][:provider].downcase == "local" ||
+ c[:packages][:fog_connection][:provider].downcase == "local")
+ Fog.mock!
+ end
+
VCAP::CloudController::Config.configure(c)
c
end
Please sign in to comment.
Something went wrong with that request. Please try again.