Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Fixed s3 url support for regions [STEAM-313]

  • Loading branch information...
commit bdcd7ee1b3ffc3be9b18a0720f1c7edad881bcec 1 parent 830cda1
@tobias tobias authored
View
77 app/models/cloud/specifics/ec2.rb
@@ -22,6 +22,29 @@ module Cloud
module Specifics
class Ec2 < Base
+ EC2_OPTIONS = {
+ 'ap-southeast-1' => {
+ :ec2_endpoint => 'ec2.ap-southeast-1.amazonaws.com',
+ :s3_endpoint => 's3-ap-southeast-1.amazonaws.com',
+ :s3_location => 'ap-southeast-1'
+ },
+ 'eu-west-1' => {
+ :ec2_endpoint => 'ec2.eu-west-1.amazonaws.com',
+ :s3_endpoint => 's3.amazonaws.com',
+ :s3_location => 'EU'
+ },
+ 'us-east-1' => {
+ :ec2_endpoint => 'ec2.us-east-1.amazonaws.com',
+ :s3_endpoint => 's3.amazonaws.com',
+ :s3_location => ''
+ },
+ 'us-west-1' => {
+ :ec2_endpoint => 'ec2.us-west-1.amazonaws.com',
+ :s3_endpoint => 's3-us-west-1.amazonaws.com',
+ :s3_location => 'us-west-1'
+ }
+ }
+
def multicast_config(instance)
{
:s3_ping => {
@@ -46,7 +69,7 @@ def launch_options(instance)
def running_instances
return [] if access_key.blank? or secret_access_key.blank?
- ec2 = Aws::Ec2.new(access_key, secret_access_key)
+ ec2 = Aws::Ec2.new(access_key, secret_access_key, :server => ec2_endpoint)
all = ec2.describe_instances.map do |instance|
instance.merge(:id => instance[:aws_instance_id],
:image => instance[:aws_image_id],
@@ -69,7 +92,10 @@ def runaway_instances
end
def unique_bucket_name(prefix)
- sc_salt = Digest::SHA1.hexdigest(Certificate.ca_certificate.certificate)
+ if prefix =~ /[^0-9a-z-]/ or prefix.length > 23
+ raise ArgumentError.new("Prefix '#{prefix}' has an invalid format. It must contain only lowercase letters, numbers, or - and have a length <= 23")
+ end
+ sc_salt = Digest::SHA1.hexdigest(Certificate.ca_certificate.certificate + @cloud_profile.provider_name)
creds_salt = Digest::SHA1.hexdigest(access_key)
suffix = Digest::SHA1.hexdigest("#{sc_salt} #{creds_salt}")
"#{prefix}#{suffix}"
@@ -132,6 +158,25 @@ def instance_run_cost(minutes, hardware_profile, region)
hours * cost_per_hour
end
+ def ec2_endpoint
+ EC2_OPTIONS[@cloud_profile.provider_name][:ec2_endpoint]
+ end
+
+ def s3_endpoint
+ EC2_OPTIONS[@cloud_profile.provider_name][:s3_endpoint]
+ end
+
+ def s3_location
+ EC2_OPTIONS[@cloud_profile.provider_name][:s3_location]
+ end
+
+ def generate_temporary_s3_url(options)
+ url = S3::Signature.generate_temporary_url(options)
+ #replace host with region specific host, since the S3 gem does
+ #not support setting it
+ url.gsub(/s3\.amazonaws\.com/, s3_endpoint)
+ end
+
protected
def access_key
@@ -162,16 +207,32 @@ def pre_signed_url(instance, options)
:bucket => s3_bucket,
:resource => s3_resource,
:expires_at => expires_at)
- S3::Signature.generate_temporary_url(options)
+ generate_temporary_s3_url(options)
end
-
+
+ {
+ :artifact_bucket_name => 'steamcannonartifacts',
+ :environment_bucket_name => 'steamcannonenvironments'
+ }.each do |bucket_name, prefix|
+ define_method bucket_name do # def artifact_bucket_name
+ if @cloud_profile.metadata[:"s3_#{bucket_name}"] # if @cloud_profile.metadata[:s3_artifact_bucket_name]
+ @cloud_profile.metadata[:"s3_#{bucket_name}"] # @cloud_profile.metadata[:s3_artifact_bucket_name]
+ else # else
+ name = unique_bucket_name(prefix) # name = unique_bucket_name('steamcannonartifacts')
+ @cloud_profile. # @cloud_profile.
+ merge_and_update_metadata(:"s3_#{bucket_name}" => name) # merge_and_update_metadata(:s3_artifact_bucket_name => name)
+ name # name
+ end # end
+ end # end
+ end
+
def multicast_bucket
- bucket_name = unique_bucket_name("SteamCannonEnvironments_")
+ bucket_name = environment_bucket_name
- s3 = Aws::S3.new(access_key, secret_access_key)
+ s3 = Aws::S3.new(access_key, secret_access_key, :server => s3_endpoint)
# Ensure our bucket exists and has correct permissions
- bucket = Aws::S3::Bucket.create(s3, bucket_name, true, 'public-read')
+ bucket = Aws::S3::Bucket.create(s3, bucket_name, true, 'public-read', :location => s3_location)
bucket_name
end
memoize :multicast_bucket
@@ -233,7 +294,7 @@ def ensure_security_group(options)
group_name = options[:name]
group_description = options[:description]
- ec2 = Aws::Ec2.new(access_key, secret_access_key)
+ ec2 = Aws::Ec2.new(access_key, secret_access_key, :server => ec2_endpoint)
begin
group = ec2.describe_security_groups([group_name])[0]
rescue Aws::AwsError => e
View
13 app/models/cloud/storage/ec2_storage.rb
@@ -25,7 +25,7 @@ def initialize(cloud_profile)
@cloud_profile = cloud_profile
@access_key = cloud_profile.username
@secret_access_key = cloud_profile.password
- @cloud_specific_hacks = cloud_profile.cloud_specific_hacks
+ @cloud_specifics = cloud_profile.cloud_specifics
end
def write(artifact_version)
@@ -50,18 +50,17 @@ def public_url(artifact_version)
:resource => path(artifact_version),
:expires_at => expires_at
}
- S3::Signature.generate_temporary_url(options)
+ @cloud_specifics.generate_temporary_s3_url(options)
end
def bucket_name
- prefix = "SteamCannonArtifacts_"
- @cloud_specific_hacks.unique_bucket_name(prefix)
+ @cloud_specifics.artifact_bucket_name
end
-
+
def bucket
# Ensure our bucket exists and has correct permissions
- s3 = Aws::S3.new(@access_key, @secret_access_key)
- bucket = Aws::S3::Bucket.create(s3, bucket_name, true, 'private')
+ s3 = Aws::S3.new(@access_key, @secret_access_key, :server => @cloud_specifics.s3_endpoint)
+ bucket = Aws::S3::Bucket.create(s3, bucket_name, true, 'private', :location => @cloud_specifics.s3_location)
bucket
end
memoize :bucket
View
14 app/models/cloud_profile.rb
@@ -17,6 +17,8 @@
# 02110-1301 USA, or see the FSF site: http://www.fsf.org.
class CloudProfile < ActiveRecord::Base
+ include HasMetadata
+
belongs_to :organization
has_many :environments
@@ -57,17 +59,9 @@ def cloud
@cloud ||= Cloud::Deltacloud.new(username, password, cloud_name, provider_name)
end
- def cloud_specific_hacks
+ def cloud_specifics
@cloud_hacks ||= Cloud::Specifics::Base.cloud_specifics(cloud_name, self)
- end
-
- def environment_bucket_name
- cloud_specific_hacks.unique_bucket_name("SteamCannonEnvironments_")
- end
-
- def artifact_bucket_name
- cloud_specific_hacks.unique_bucket_name("SteamCannonArtifacts_")
- end
+ end
protected
def encrypt_password
View
8 app/views/cloud_profiles/show.html.haml
@@ -22,12 +22,12 @@
%b Password:
= h @cloud_profile.obfuscated_password
- - if @cloud_profile.artifact_bucket_name
+ - if @cloud_profile.cloud_specifics.artifact_bucket_name
%p
%b Amazon S3 Artifact Bucket:
- &= @cloud_profile.artifact_bucket_name
- - if @cloud_profile.environment_bucket_name
+ &= @cloud_profile.cloud_specifics.artifact_bucket_name
+ - if @cloud_profile.cloud_specifics.environment_bucket_name
%p
%b Amazon Environment Clustering Bucket:
- &= @cloud_profile.environment_bucket_name
+ &= @cloud_profile.cloud_specifics.environment_bucket_name
View
125 spec/models/cloud/specifics/ec2_spec.rb
@@ -22,7 +22,9 @@
before(:each) do
@cloud_profile = Factory.build(:cloud_profile,
:username => 'username',
- :password => 'password')
+ :password => 'password',
+ :provider_name => 'eu-west-1',
+ :cloud_name => 'ec2')
@ec2 = Cloud::Specifics::Ec2.new(@cloud_profile)
@instance = Factory.build(:instance)
end
@@ -127,36 +129,28 @@
it "should get access_key from cloud_profile object" do
@cloud_profile.should_receive(:username).and_return('username')
- @sig.should_receive(:generate_temporary_url).
- with(hash_including(:access_key => 'username'))
- @ec2.send(:pre_signed_url, @instance, {})
+ verify_signature_contains(:access_key => 'username')
end
it "should get secret_access_key from cloud_profile object" do
@cloud_profile.should_receive(:password).and_return('password')
- @sig.should_receive(:generate_temporary_url).
- with(hash_including(:secret_access_key => 'password'))
- @ec2.send(:pre_signed_url, @instance, {})
+ verify_signature_contains(:secret_access_key => 'password')
end
it "should get multicast bucket" do
@ec2.should_receive(:multicast_bucket).and_return('bucket')
- @sig.should_receive(:generate_temporary_url).
- with(hash_including(:bucket => 'bucket'))
- @ec2.send(:pre_signed_url, @instance, {})
+ verify_signature_contains(:bucket => 'bucket')
end
it "should expire 1 year after instance creation" do
expires_at = @created_at + 1.year
@instance.should_receive(:created_at).and_return(@created_at)
- @sig.should_receive(:generate_temporary_url).
- with(hash_including(:expires_at => expires_at))
- @ec2.send(:pre_signed_url, @instance, {})
+ verify_signature_contains(:expires_at => expires_at)
end
- it "should return temporary url" do
- @sig.should_receive(:generate_temporary_url).and_return('url')
- @ec2.send(:pre_signed_url, @instance, {}).should == 'url'
+ def verify_signature_contains(options)
+ @ec2.should_receive(:generate_temporary_s3_url).with(hash_including(options)).and_return('a-url')
+ @ec2.send(:pre_signed_url, @instance, { })
end
end
@@ -167,32 +161,45 @@
@s3::Bucket.stub(:create)
end
- it "should generate suffix from username and ca certificate" do
+ it "should generate suffix from username, ca certificate, and region" do
ca_certificate = Factory(:certificate)
Certificate.should_receive(:ca_certificate).and_return(ca_certificate)
ca_certificate.should_receive(:certificate).and_return('certificate')
Digest::SHA1.should_receive(:hexdigest).with('username')
- Digest::SHA1.should_receive(:hexdigest).with('certificate')
+ Digest::SHA1.should_receive(:hexdigest).with('certificateeu-west-1')
Digest::SHA1.should_receive(:hexdigest).and_return('hexdigest')
@ec2.send(:multicast_bucket)
end
it "should create a new s3 object" do
- @s3.should_receive(:new).with('username', 'password')
+ @s3.should_receive(:new).with('username', 'password', :server => 's3.amazonaws.com')
@ec2.send(:multicast_bucket)
end
it "should create s3 bucket with public read permissions" do
- @s3::Bucket.should_receive(:create).with(anything, anything, true, 'public-read')
+ @s3::Bucket.should_receive(:create).with(anything, anything, true, 'public-read', :location => 'EU')
@ec2.send(:multicast_bucket)
end
it "should return bucket name" do
Digest::SHA1.stub!(:hexdigest).and_return('suffix')
- @ec2.send(:multicast_bucket).should == "SteamCannonEnvironments_suffix"
+ @ec2.send(:multicast_bucket).should == "steamcannonenvironmentssuffix"
end
end
+ describe "generate_temporary_s3_url" do
+ it "should use S3::Signature.generate_temporary_url" do
+ S3::Signature.should_receive(:generate_temporary_url).with('the-options').and_return('')
+ @ec2.send(:generate_temporary_s3_url, 'the-options')
+ end
+
+ it "should set the region specific host in the url" do
+ S3::Signature.stub(:generate_temporary_url).and_return('http://s3.amazonaws.com/')
+ @ec2.should_receive(:s3_endpoint).and_return('a-different-host')
+ @ec2.send(:generate_temporary_s3_url, @artifact_version).should =~ /a-different-host/
+ end
+ end
+
describe "base_security_group" do
it "should be named steamcannon" do
@ec2.send(:base_security_group)[:name].should == 'steamcannon'
@@ -312,4 +319,80 @@
end
end
end
+
+ describe "ec2_endpoint" do
+ it "should use the provider from the profile to lookup the url" do
+ @cloud_profile.should_receive(:provider_name).and_return('us-east-1')
+ @ec2.ec2_endpoint.should == 'ec2.us-east-1.amazonaws.com'
+ end
+ end
+
+ describe "s3_endpoint" do
+ it "should use the provider from the profile to lookup the url" do
+ @cloud_profile.should_receive(:provider_name).and_return('ap-southeast-1')
+ @ec2.s3_endpoint.should == 's3-ap-southeast-1.amazonaws.com'
+ end
+ end
+
+ describe "s3_location" do
+ it "should use the provider from the profile to lookup the location constraint" do
+ @cloud_profile.should_receive(:provider_name).and_return('eu-west-1')
+ @ec2.s3_location.should == 'EU'
+ end
+ end
+
+ describe 'unique_bucket_name' do
+ it "should raise if the prefix length is too long" do
+ lambda {
+ @ec2.unique_bucket_name('x' * 24)
+ }.should raise_error(ArgumentError)
+ end
+
+ it "should raise if the prefix contains uppercase characters" do
+ lambda {
+ @ec2.unique_bucket_name('ABC')
+ }.should raise_error(ArgumentError)
+ end
+
+ it "should raise if the prefix contains invalid characters" do
+ lambda {
+ @ec2.unique_bucket_name('a_b')
+ }.should raise_error(ArgumentError)
+ end
+
+ it "should not raise when given a properly formatted prefix" do
+ lambda {
+ @ec2.unique_bucket_name(('x' * 21) + '-1')
+ }.should_not raise_error
+ end
+ end
+
+ {
+ :artifact => 'steamcannonartifacts',
+ :environment => 'steamcannonenvironments'
+ }.each do |type, prefix|
+ describe type do
+ before(:each) do
+ @metadata = { }
+ @cloud_profile.stub(:metadata).and_return(@metadata)
+ @ec2.stub(:unique_bucket_name).and_return('a-bucket')
+ end
+
+ it "should ask for a unique bucket name" do
+ @ec2.should_receive(:unique_bucket_name).with(prefix).and_return('a-bucket')
+ @ec2.send("#{type}_bucket_name")
+ end
+
+ it "should use the value stored in the cloud_profile if available" do
+ @metadata[:"s3_#{type}_bucket_name"] = 'a-bucket'
+ @ec2.should_not_receive(:unique_bucket_name)
+ @ec2.send("#{type}_bucket_name").should == 'a-bucket'
+ end
+
+ it "should store the bucket name in the cloud profile's metadata" do
+ @cloud_profile.should_receive(:merge_and_update_metadata).with(:"s3_#{type}_bucket_name" => 'a-bucket')
+ @ec2.send("#{type}_bucket_name")
+ end
+ end
+ end
end
View
39 spec/models/cloud/storage/ec2_storage_spec.rb
@@ -20,11 +20,11 @@
describe Cloud::Storage::Ec2Storage do
before(:each) do
- @cloud_specific_hacks = mock('hacks')
+ @cloud_specifics = mock('cloud_specifics')
@cloud_profile = mock_model(CloudProfile,
:username => 'access_key',
:password => 'secret_access_key',
- :cloud_specific_hacks => @cloud_specific_hacks)
+ :cloud_specifics => @cloud_specifics)
@ec2 = Cloud::Storage::Ec2Storage.new(@cloud_profile)
@artifact_version = Factory.build(:artifact_version)
@path = 'path/to/file.war'
@@ -68,8 +68,8 @@
describe "public_url" do
before(:each) do
- @sig = S3::Signature
@ec2.stub!(:bucket_name).and_return('name')
+ @cloud_specifics.stub!(:s3_endpoint).and_return('s3-endpoint')
end
it "should generate url for cloud credentials" do
@@ -88,9 +88,9 @@
it "should generate url for correct resource" do
verify_signature_contains(:resource => @path)
end
-
+
def verify_signature_contains(options)
- @sig.should_receive(:generate_temporary_url).with(hash_including(options))
+ @cloud_specifics.should_receive(:generate_temporary_s3_url).with(hash_including(options)).and_return('a-url')
@ec2.public_url(@artifact_version)
end
end
@@ -99,30 +99,37 @@ def verify_signature_contains(options)
before(:each) do
@bucket = Aws::S3::Bucket
@bucket.stub!(:create)
- @cloud_specific_hacks.stub!(:unique_bucket_name).and_return('')
+ @cloud_specifics.stub!(:unique_bucket_name).and_return('')
+ @cloud_specifics.stub!(:s3_location).and_return('')
+ @cloud_specifics.stub!(:s3_endpoint).and_return('s3.local')
+ @cloud_specifics.stub!(:artifact_bucket_name).and_return('bucket')
end
- it "should create with correct prefix" do
- prefix = "SteamCannonArtifacts_"
- @cloud_specific_hacks.should_receive(:unique_bucket_name).with(prefix).and_return(prefix)
- @bucket.should_receive(:create).with(anything, /^#{prefix}/, anything, anything)
+ it "should provide the s3_endpoint when creating the s3 connection" do
+ @cloud_specifics.should_receive(:s3_endpoint).and_return('s3.local')
+ Aws::S3.should_receive(:new).with('access_key', 'secret_access_key', :server => 's3.local')
+ @ec2.bucket
+ end
+
+ it "should use the bucket name from the cloud_specifics" do
+ @cloud_specifics.should_receive(:artifact_bucket_name).and_return('bucket')
+ @bucket.should_receive(:create).with(anything, 'bucket', anything, anything, anything)
@ec2.bucket
end
it "should create if it doesn't already exist" do
- @bucket.should_receive(:create).with(anything, anything, true, anything)
+ @bucket.should_receive(:create).with(anything, anything, true, anything, anything)
@ec2.bucket
end
it "should create with private permissions" do
- @bucket.should_receive(:create).with(anything, anything, anything, 'private')
+ @bucket.should_receive(:create).with(anything, anything, anything, 'private', anything)
@ec2.bucket
end
- it "should create a unique bucket name" do
- suffix = /unique_bucket_name$/
- @cloud_specific_hacks.should_receive(:unique_bucket_name).and_return('unique_bucket_name')
- @bucket.should_receive(:create).with(anything, suffix, anything, anything)
+ it "should create with the correct location constraint" do
+ @cloud_specifics.should_receive(:s3_location).and_return('GOOP')
+ @bucket.should_receive(:create).with(anything, anything, anything, anything, :location => 'GOOP')
@ec2.bucket
end
end
Please sign in to comment.
Something went wrong with that request. Please try again.