diff --git a/Gemfile.lock b/Gemfile.lock index 61c06419c..f9e64b445 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,19 +1,27 @@ +PATH + remote: . + specs: + paperclip (2.4.5) + activerecord (>= 2.3.0) + activesupport (>= 2.3.2) + cocaine (>= 0.0.2) + mime-types + GEM remote: http://rubygems.org/ specs: - activemodel (3.1.0) - activesupport (= 3.1.0) - bcrypt-ruby (~> 3.0.0) + activemodel (3.1.2) + activesupport (= 3.1.2) builder (~> 3.0.0) i18n (~> 0.6) - activerecord (3.1.0) - activemodel (= 3.1.0) - activesupport (= 3.1.0) + activerecord (3.1.2) + activemodel (= 3.1.2) + activesupport (= 3.1.2) arel (~> 2.2.1) tzinfo (~> 0.3.29) - activesupport (3.1.0) + activesupport (3.1.2) multi_json (~> 1.0) - appraisal (0.3.8) + appraisal (0.4.0) bundler rake arel (2.2.1) @@ -23,13 +31,13 @@ GEM cucumber (>= 1.0.2) rdiscount (>= 1.6.8) rspec (>= 2.6.0) - aws-s3 (0.6.2) - builder - mime-types - xml-simple + aws-sdk (1.2.1) + httparty (~> 0.7) + json (~> 1.4) + nokogiri (>= 1.4.4) + uuidtools (~> 2.1) bcat (0.6.2) rack (~> 1.0) - bcrypt-ruby (3.0.1) builder (3.0.0) capybara (1.1.1) mime-types (>= 1.16) @@ -41,7 +49,6 @@ GEM childprocess (0.2.2) ffi (~> 1.0.6) cocaine (0.2.0) - coderay (1.0.0) cucumber (1.0.6) builder (>= 2.1.2) diff-lcs (>= 1.1.2) @@ -65,31 +72,27 @@ GEM formatador (0.2.1) gherkin (2.4.21) json (>= 1.4.6) + httparty (0.8.1) + multi_json + multi_xml i18n (0.6.0) json (1.6.1) json_pure (1.6.1) metaclass (0.0.1) - method_source (0.6.5) - ruby_parser (>= 2.0.5) mime-types (1.16) mocha (0.10.0) metaclass (~> 0.0.1) multi_json (1.0.3) + multi_xml (0.4.1) net-scp (1.0.4) net-ssh (>= 1.99.1) net-ssh (2.1.4) nokogiri (1.5.0) - pry (0.9.6) - coderay (>= 0.9.8) - method_source (>= 0.6.5) - ruby_parser (>= 2.0.5) - slop (~> 2.1.0) rack (1.3.3) rack-test (0.6.1) rack (>= 1.0) rake (0.9.2) rdiscount (1.6.8) - rdoc (3.9.4) rspec (2.6.0) rspec-core (~> 2.6.0) rspec-expectations (~> 2.6.0) @@ -99,21 +102,17 @@ GEM diff-lcs (~> 1.1.2) rspec-mocks (2.6.0) ruby-hmac (0.4.0) - ruby_parser (2.3.1) - sexp_processor (~> 3.0) rubyzip (0.9.4) selenium-webdriver (2.7.0) childprocess (>= 0.2.1) ffi (>= 1.0.7) json_pure rubyzip - sexp_processor (3.0.7) shoulda (2.11.3) - slop (2.1.0) sqlite3 (1.3.4) term-ansicolor (1.0.6) - tzinfo (0.3.29) - xml-simple (1.1.0) + tzinfo (0.3.31) + uuidtools (2.1.2) xpath (0.1.4) nokogiri (~> 1.3) @@ -121,10 +120,9 @@ PLATFORMS ruby DEPENDENCIES - activerecord - appraisal + appraisal (~> 0.4.0) aruba - aws-s3 + aws-sdk bundler capybara cocaine (~> 0.2) @@ -132,10 +130,8 @@ DEPENDENCIES fakeweb fog jruby-openssl - mime-types mocha - pry + paperclip! rake - rdoc shoulda sqlite3 (~> 1.3.4) diff --git a/features/step_definitions/rails_steps.rb b/features/step_definitions/rails_steps.rb index a88bba1e8..a1af826c3 100644 --- a/features/step_definitions/rails_steps.rb +++ b/features/step_definitions/rails_steps.rb @@ -10,7 +10,7 @@ gem "sqlite3" gem "capybara" gem "gherkin" - gem "aws-s3" + gem "aws-sdk" """ And I configure the application to use "paperclip" from this project And I reset Bundler environment variable diff --git a/features/step_definitions/s3_steps.rb b/features/step_definitions/s3_steps.rb index fb8bb3825..a8e034861 100644 --- a/features/step_definitions/s3_steps.rb +++ b/features/step_definitions/s3_steps.rb @@ -1,6 +1,6 @@ When /^I attach the file "([^"]*)" to "([^"]*)" on S3$/ do |file_path, field| definition = User.attachment_definitions[field.downcase.to_sym] - path = "http://s3.amazonaws.com/paperclip#{definition[:path]}" + path = "https://paperclip.s3.amazonaws.com#{definition[:path]}" path.gsub!(':filename', File.basename(file_path)) path.gsub!(/:([^\/\.]+)/) do |match| "([^\/\.]+)" diff --git a/gemfiles/rails2.gemfile.lock b/gemfiles/rails2.gemfile.lock deleted file mode 100644 index 9b3ae37cd..000000000 --- a/gemfiles/rails2.gemfile.lock +++ /dev/null @@ -1,136 +0,0 @@ -PATH - remote: ../ - specs: - paperclip (2.4.5) - activerecord (>= 2.3.0) - activesupport (>= 2.3.2) - cocaine (>= 0.0.2) - mime-types - -GEM - remote: http://rubygems.org/ - specs: - actionmailer (2.3.14) - actionpack (= 2.3.14) - actionpack (2.3.14) - activesupport (= 2.3.14) - rack (~> 1.1.0) - activerecord (2.3.14) - activesupport (= 2.3.14) - activeresource (2.3.14) - activesupport (= 2.3.14) - activesupport (2.3.14) - appraisal (0.4.0) - bundler - rake - aruba (0.4.6) - bcat (>= 0.6.1) - childprocess (>= 0.2.0) - cucumber (>= 1.0.2) - rdiscount (>= 1.6.8) - rspec (>= 2.6.0) - aws-s3 (0.6.2) - builder - mime-types - xml-simple - bcat (0.6.2) - rack (~> 1.0) - builder (3.0.0) - capybara (1.1.1) - mime-types (>= 1.16) - nokogiri (>= 1.3.3) - rack (>= 1.0.0) - rack-test (>= 0.5.4) - selenium-webdriver (~> 2.0) - xpath (~> 0.1.4) - childprocess (0.2.2) - ffi (~> 1.0.6) - cocaine (0.2.0) - cucumber (1.0.6) - builder (>= 2.1.2) - diff-lcs (>= 1.1.2) - gherkin (~> 2.4.18) - json (>= 1.4.6) - term-ansicolor (>= 1.0.6) - diff-lcs (1.1.3) - excon (0.7.4) - fakeweb (1.3.0) - ffi (1.0.9) - fog (1.0.0) - builder - excon (~> 0.7.3) - formatador (~> 0.2.0) - mime-types - multi_json (~> 1.0.3) - net-scp (~> 1.0.4) - net-ssh (~> 2.1.4) - nokogiri (~> 1.5.0) - ruby-hmac - formatador (0.2.1) - gherkin (2.4.21) - json (>= 1.4.6) - json (1.6.1) - json_pure (1.6.1) - metaclass (0.0.1) - mime-types (1.16) - mocha (0.10.0) - metaclass (~> 0.0.1) - multi_json (1.0.3) - net-scp (1.0.4) - net-ssh (>= 1.99.1) - net-ssh (2.1.4) - nokogiri (1.5.0) - rack (1.1.2) - rack-test (0.6.1) - rack (>= 1.0) - rails (2.3.14) - actionmailer (= 2.3.14) - actionpack (= 2.3.14) - activerecord (= 2.3.14) - activeresource (= 2.3.14) - activesupport (= 2.3.14) - rake (>= 0.8.3) - rake (0.9.2) - rdiscount (1.6.8) - rspec (2.6.0) - rspec-core (~> 2.6.0) - rspec-expectations (~> 2.6.0) - rspec-mocks (~> 2.6.0) - rspec-core (2.6.4) - rspec-expectations (2.6.0) - diff-lcs (~> 1.1.2) - rspec-mocks (2.6.0) - ruby-hmac (0.4.0) - rubyzip (0.9.4) - selenium-webdriver (2.7.0) - childprocess (>= 0.2.1) - ffi (>= 1.0.7) - json_pure - rubyzip - shoulda (2.11.3) - sqlite3 (1.3.4) - term-ansicolor (1.0.6) - xml-simple (1.1.0) - xpath (0.1.4) - nokogiri (~> 1.3) - -PLATFORMS - ruby - -DEPENDENCIES - appraisal (~> 0.4.0) - aruba - aws-s3 - bundler - capybara - cocaine (~> 0.2) - cucumber (~> 1.0.0) - fakeweb - fog - jruby-openssl - mocha - paperclip! - rails (~> 2.3.14) - rake - shoulda - sqlite3 (~> 1.3.4) diff --git a/gemfiles/rails3.gemfile.lock b/gemfiles/rails3.gemfile.lock deleted file mode 100644 index 6c2735431..000000000 --- a/gemfiles/rails3.gemfile.lock +++ /dev/null @@ -1,178 +0,0 @@ -PATH - remote: ../ - specs: - paperclip (2.4.5) - activerecord (>= 2.3.0) - activesupport (>= 2.3.2) - cocaine (>= 0.0.2) - mime-types - -GEM - remote: http://rubygems.org/ - specs: - abstract (1.0.0) - actionmailer (3.0.10) - actionpack (= 3.0.10) - mail (~> 2.2.19) - actionpack (3.0.10) - activemodel (= 3.0.10) - activesupport (= 3.0.10) - builder (~> 2.1.2) - erubis (~> 2.6.6) - i18n (~> 0.5.0) - rack (~> 1.2.1) - rack-mount (~> 0.6.14) - rack-test (~> 0.5.7) - tzinfo (~> 0.3.23) - activemodel (3.0.10) - activesupport (= 3.0.10) - builder (~> 2.1.2) - i18n (~> 0.5.0) - activerecord (3.0.10) - activemodel (= 3.0.10) - activesupport (= 3.0.10) - arel (~> 2.0.10) - tzinfo (~> 0.3.23) - activeresource (3.0.10) - activemodel (= 3.0.10) - activesupport (= 3.0.10) - activesupport (3.0.10) - appraisal (0.4.0) - bundler - rake - arel (2.0.10) - aruba (0.4.6) - bcat (>= 0.6.1) - childprocess (>= 0.2.0) - cucumber (>= 1.0.2) - rdiscount (>= 1.6.8) - rspec (>= 2.6.0) - aws-s3 (0.6.2) - builder - mime-types - xml-simple - bcat (0.6.2) - rack (~> 1.0) - builder (2.1.2) - capybara (1.1.1) - mime-types (>= 1.16) - nokogiri (>= 1.3.3) - rack (>= 1.0.0) - rack-test (>= 0.5.4) - selenium-webdriver (~> 2.0) - xpath (~> 0.1.4) - childprocess (0.2.2) - ffi (~> 1.0.6) - cocaine (0.2.0) - cucumber (1.0.6) - builder (>= 2.1.2) - diff-lcs (>= 1.1.2) - gherkin (~> 2.4.18) - json (>= 1.4.6) - term-ansicolor (>= 1.0.6) - diff-lcs (1.1.3) - erubis (2.6.6) - abstract (>= 1.0.0) - excon (0.7.4) - fakeweb (1.3.0) - ffi (1.0.9) - fog (1.0.0) - builder - excon (~> 0.7.3) - formatador (~> 0.2.0) - mime-types - multi_json (~> 1.0.3) - net-scp (~> 1.0.4) - net-ssh (~> 2.1.4) - nokogiri (~> 1.5.0) - ruby-hmac - formatador (0.2.1) - gherkin (2.4.21) - json (>= 1.4.6) - i18n (0.5.0) - json (1.6.1) - json_pure (1.6.1) - mail (2.2.19) - activesupport (>= 2.3.6) - i18n (>= 0.4.0) - mime-types (~> 1.16) - treetop (~> 1.4.8) - metaclass (0.0.1) - mime-types (1.16) - mocha (0.10.0) - metaclass (~> 0.0.1) - multi_json (1.0.3) - net-scp (1.0.4) - net-ssh (>= 1.99.1) - net-ssh (2.1.4) - nokogiri (1.5.0) - polyglot (0.3.2) - rack (1.2.4) - rack-mount (0.6.14) - rack (>= 1.0.0) - rack-test (0.5.7) - rack (>= 1.0) - rails (3.0.10) - actionmailer (= 3.0.10) - actionpack (= 3.0.10) - activerecord (= 3.0.10) - activeresource (= 3.0.10) - activesupport (= 3.0.10) - bundler (~> 1.0) - railties (= 3.0.10) - railties (3.0.10) - actionpack (= 3.0.10) - activesupport (= 3.0.10) - rake (>= 0.8.7) - rdoc (~> 3.4) - thor (~> 0.14.4) - rake (0.9.2) - rdiscount (1.6.8) - rdoc (3.9.4) - rspec (2.6.0) - rspec-core (~> 2.6.0) - rspec-expectations (~> 2.6.0) - rspec-mocks (~> 2.6.0) - rspec-core (2.6.4) - rspec-expectations (2.6.0) - diff-lcs (~> 1.1.2) - rspec-mocks (2.6.0) - ruby-hmac (0.4.0) - rubyzip (0.9.4) - selenium-webdriver (2.7.0) - childprocess (>= 0.2.1) - ffi (>= 1.0.7) - json_pure - rubyzip - shoulda (2.11.3) - sqlite3 (1.3.4) - term-ansicolor (1.0.6) - thor (0.14.6) - treetop (1.4.10) - polyglot - polyglot (>= 0.3.1) - tzinfo (0.3.30) - xml-simple (1.1.0) - xpath (0.1.4) - nokogiri (~> 1.3) - -PLATFORMS - ruby - -DEPENDENCIES - appraisal (~> 0.4.0) - aruba - aws-s3 - bundler - capybara - cocaine (~> 0.2) - cucumber (~> 1.0.0) - fakeweb - fog - jruby-openssl - mocha - paperclip! - rails (~> 3.0.10) - rake - shoulda - sqlite3 (~> 1.3.4) diff --git a/gemfiles/rails3_1.gemfile.lock b/gemfiles/rails3_1.gemfile.lock deleted file mode 100644 index 642ecd207..000000000 --- a/gemfiles/rails3_1.gemfile.lock +++ /dev/null @@ -1,190 +0,0 @@ -PATH - remote: ../ - specs: - paperclip (2.4.5) - activerecord (>= 2.3.0) - activesupport (>= 2.3.2) - cocaine (>= 0.0.2) - mime-types - -GEM - remote: http://rubygems.org/ - specs: - actionmailer (3.1.0) - actionpack (= 3.1.0) - mail (~> 2.3.0) - actionpack (3.1.0) - activemodel (= 3.1.0) - activesupport (= 3.1.0) - builder (~> 3.0.0) - erubis (~> 2.7.0) - i18n (~> 0.6) - rack (~> 1.3.2) - rack-cache (~> 1.0.3) - rack-mount (~> 0.8.2) - rack-test (~> 0.6.1) - sprockets (~> 2.0.0) - activemodel (3.1.0) - activesupport (= 3.1.0) - bcrypt-ruby (~> 3.0.0) - builder (~> 3.0.0) - i18n (~> 0.6) - activerecord (3.1.0) - activemodel (= 3.1.0) - activesupport (= 3.1.0) - arel (~> 2.2.1) - tzinfo (~> 0.3.29) - activeresource (3.1.0) - activemodel (= 3.1.0) - activesupport (= 3.1.0) - activesupport (3.1.0) - multi_json (~> 1.0) - appraisal (0.4.0) - bundler - rake - arel (2.2.1) - aruba (0.4.6) - bcat (>= 0.6.1) - childprocess (>= 0.2.0) - cucumber (>= 1.0.2) - rdiscount (>= 1.6.8) - rspec (>= 2.6.0) - aws-s3 (0.6.2) - builder - mime-types - xml-simple - bcat (0.6.2) - rack (~> 1.0) - bcrypt-ruby (3.0.1) - builder (3.0.0) - capybara (1.1.1) - mime-types (>= 1.16) - nokogiri (>= 1.3.3) - rack (>= 1.0.0) - rack-test (>= 0.5.4) - selenium-webdriver (~> 2.0) - xpath (~> 0.1.4) - childprocess (0.2.2) - ffi (~> 1.0.6) - cocaine (0.2.0) - cucumber (1.0.6) - builder (>= 2.1.2) - diff-lcs (>= 1.1.2) - gherkin (~> 2.4.18) - json (>= 1.4.6) - term-ansicolor (>= 1.0.6) - diff-lcs (1.1.3) - erubis (2.7.0) - excon (0.7.4) - fakeweb (1.3.0) - ffi (1.0.9) - fog (1.0.0) - builder - excon (~> 0.7.3) - formatador (~> 0.2.0) - mime-types - multi_json (~> 1.0.3) - net-scp (~> 1.0.4) - net-ssh (~> 2.1.4) - nokogiri (~> 1.5.0) - ruby-hmac - formatador (0.2.1) - gherkin (2.4.21) - json (>= 1.4.6) - hike (1.2.1) - i18n (0.6.0) - json (1.6.1) - json_pure (1.6.1) - mail (2.3.0) - i18n (>= 0.4.0) - mime-types (~> 1.16) - treetop (~> 1.4.8) - metaclass (0.0.1) - mime-types (1.16) - mocha (0.10.0) - metaclass (~> 0.0.1) - multi_json (1.0.3) - net-scp (1.0.4) - net-ssh (>= 1.99.1) - net-ssh (2.1.4) - nokogiri (1.5.0) - polyglot (0.3.2) - rack (1.3.3) - rack-cache (1.0.3) - rack (>= 0.4) - rack-mount (0.8.3) - rack (>= 1.0.0) - rack-ssl (1.3.2) - rack - rack-test (0.6.1) - rack (>= 1.0) - rails (3.1.0) - actionmailer (= 3.1.0) - actionpack (= 3.1.0) - activerecord (= 3.1.0) - activeresource (= 3.1.0) - activesupport (= 3.1.0) - bundler (~> 1.0) - railties (= 3.1.0) - railties (3.1.0) - actionpack (= 3.1.0) - activesupport (= 3.1.0) - rack-ssl (~> 1.3.2) - rake (>= 0.8.7) - rdoc (~> 3.4) - thor (~> 0.14.6) - rake (0.9.2) - rdiscount (1.6.8) - rdoc (3.9.4) - rspec (2.6.0) - rspec-core (~> 2.6.0) - rspec-expectations (~> 2.6.0) - rspec-mocks (~> 2.6.0) - rspec-core (2.6.4) - rspec-expectations (2.6.0) - diff-lcs (~> 1.1.2) - rspec-mocks (2.6.0) - ruby-hmac (0.4.0) - rubyzip (0.9.4) - selenium-webdriver (2.7.0) - childprocess (>= 0.2.1) - ffi (>= 1.0.7) - json_pure - rubyzip - shoulda (2.11.3) - sprockets (2.0.1) - hike (~> 1.2) - rack (~> 1.0) - tilt (~> 1.1, != 1.3.0) - sqlite3 (1.3.4) - term-ansicolor (1.0.6) - thor (0.14.6) - tilt (1.3.3) - treetop (1.4.10) - polyglot - polyglot (>= 0.3.1) - tzinfo (0.3.30) - xml-simple (1.1.0) - xpath (0.1.4) - nokogiri (~> 1.3) - -PLATFORMS - ruby - -DEPENDENCIES - appraisal (~> 0.4.0) - aruba - aws-s3 - bundler - capybara - cocaine (~> 0.2) - cucumber (~> 1.0.0) - fakeweb - fog - jruby-openssl - mocha - paperclip! - rails (~> 3.1.0) - rake - shoulda - sqlite3 (~> 1.3.4) diff --git a/lib/paperclip/storage/s3.rb b/lib/paperclip/storage/s3.rb index 4c269a911..05f276b93 100644 --- a/lib/paperclip/storage/s3.rb +++ b/lib/paperclip/storage/s3.rb @@ -67,12 +67,21 @@ module Storage # S3 (strictly speaking) does not support directories, you can still use a / to # separate parts of your file name. # * +s3_host_name+: If you are using your bucket in Tokyo region etc, write host_name. + # * +s3_metadata+: These key/value pairs will be stored with the + # object. This option works by prefixing each key with + # "x-amz-meta-" before sending it as a header on the object + # upload request. + # * +s3_storage_class+: If this option is set to + # :reduced_redundancy, the object will be stored using Reduced + # Redundancy Storage. RRS enables customers to reduce their + # costs by storing non-critical, reproducible data at lower + # levels of redundancy than Amazon S3's standard storage. module S3 def self.extended base begin require 'aws/s3' rescue LoadError => e - e.message << " (You may need to install the aws-s3 gem)" + e.message << " (You may need to install the aws-sdk gem)" raise e end unless defined?(AWS::S3) @@ -85,7 +94,19 @@ def self.extended base permission = permission.call(attachment, style) if permission.is_a?(Proc) (permission == :public_read) ? 'http' : 'https' end - @s3_headers = @options[:s3_headers] || {} + @s3_metadata = @options[:s3_metadata] || {} + @s3_headers = (@options[:s3_headers] || {}).inject({}) do |headers,(name,value)| + case name.to_s + when /^x-amz-meta-(.*)/i + @s3_metadata[$1.downcase] = value + else + name = name.to_s.downcase.sub(/^x-amz-/,'').tr("-","_").to_sym + headers[name] = value + end + headers + end + + @s3_headers[:storage_class] = @options[:s3_storage_class] if @options[:s3_storage_class] unless @options[:url].to_s.match(/^:s3.*url$/) || @options[:url] == ":asset_host" @options[:path] = @options[:path].gsub(/:url/, @options[:url]).gsub(/^:rails_root\/public\/system/, '') @@ -94,9 +115,6 @@ def self.extended base @options[:url] = @options[:url].inspect if @options[:url].is_a?(Symbol) @http_proxy = @options[:http_proxy] || nil - if @http_proxy - @s3_options.merge!({:proxy => @http_proxy}) - end end Paperclip.interpolates(:s3_alias_url) do |attachment, style| "#{attachment.s3_protocol(style)}://#{attachment.s3_host_alias}/#{attachment.path(style).gsub(%r{^/}, "")}" @@ -113,7 +131,9 @@ def self.extended base end def expiring_url(time = 3600, style_name = default_style) - path.nil? ? nil : s3_object.url_for(path(style_name), bucket_name, :expires_in => time, :use_ssl => (s3_protocol(style_name) == 'https')) + if path + s3_object(style_name).url_for(:read, :expires => time, :secure => use_secure_protocol?(style_name)).to_s + end end def s3_credentials @@ -133,7 +153,39 @@ def s3_host_alias def bucket_name @bucket = @options[:bucket] || s3_credentials[:bucket] @bucket = @bucket.call(self) if @bucket.is_a?(Proc) - @bucket + @bucket or raise ArgumentError, "missing required :bucket option" + end + + def s3_interface + @s3_interface ||= begin + config = { :s3_endpoint => s3_host_name } + + if using_http_proxy? + + proxy_opts = { :host => http_proxy_host } + proxy_opts[:port] = http_proxy_port if http_proxy_port + if http_proxy_user + userinfo = http_proxy_user.to_s + userinfo += ":#{http_proxy_password}" if http_proxy_password + proxy_opts[:userinfo] = userinfo + end + config[:proxy_uri] = URI::HTTP.build(proxy_opts) + end + + [:access_key_id, :secret_access_key].each do |opt| + config[opt] = s3_credentials[opt] if s3_credentials[opt] + end + + AWS::S3.new(config.merge(@s3_options)) + end + end + + def s3_bucket + @s3_bucket ||= s3_interface.buckets[bucket_name] + end + + def s3_object style_name = default_style + s3_bucket.objects[path(style_name).sub(%r{^/},'')] end def using_http_proxy? @@ -173,7 +225,7 @@ def parse_credentials creds def exists?(style = default_style) if original_filename - s3_object.exists?(path(style), bucket_name) + s3_object(style).exists? else false end @@ -202,30 +254,31 @@ def to_file style = default_style basename = File.basename(filename, extname) file = Tempfile.new([basename, extname]) file.binmode - file.write(s3_object.value(path(style), bucket_name)) + file.write(s3_object(style).read) file.rewind return file end def create_bucket - s3_bucket.create(bucket_name) + s3_interface.buckets.create(bucket_name) end def flush_writes #:nodoc: @queued_for_write.each do |style, file| begin log("saving #{path(style)}") - s3_object.store(path(style), - file, - bucket_name, - {:content_type => file.content_type.to_s.strip, - :access => s3_permissions(style), - }.merge(@s3_headers)) - rescue AWS::S3::NoSuchBucket => e + acl = @s3_permissions[style] || @s3_permissions[:default] + acl = acl.call(self, style) if acl.respond_to?(:call) + write_options = { + :content_type => file.content_type.to_s.strip, + :acl => acl + } + write_options[:metadata] = @s3_metadata unless @s3_metadata.empty? + write_options.merge!(@s3_headers) + s3_object(style).write(file, write_options) + rescue AWS::S3::Errors::NoSuchBucket => e create_bucket retry - rescue AWS::S3::ResponseError => e - raise end end @@ -238,8 +291,8 @@ def flush_deletes #:nodoc: @queued_for_delete.each do |path| begin log("deleting #{path}") - s3_object.delete(path, bucket_name) - rescue AWS::S3::ResponseError + s3_bucket.objects[path.sub(%r{^/},'')].delete + rescue AWS::Errors::Base => e # Ignore this. end end @@ -260,18 +313,6 @@ def find_credentials creds end private :find_credentials - def s3_object - establish_connection! - AWS::S3::S3Object - end - private :s3_object - - def s3_bucket - establish_connection! - AWS::S3::Bucket - end - private :s3_bucket - def establish_connection! @connection ||= AWS::S3::Base.establish_connection!( @s3_options.merge( :access_key_id => s3_credentials[:access_key_id], @@ -279,6 +320,11 @@ def establish_connection! )) end private :establish_connection! + + def use_secure_protocol?(style_name) + s3_protocol(style_name) == "https" + end + private :use_secure_protocol? end end end diff --git a/paperclip.gemspec b/paperclip.gemspec index ca97759f3..4006a2fc8 100644 --- a/paperclip.gemspec +++ b/paperclip.gemspec @@ -28,7 +28,7 @@ Gem::Specification.new do |s| s.add_development_dependency('shoulda') s.add_development_dependency('appraisal', '~> 0.4.0') s.add_development_dependency('mocha') - s.add_development_dependency('aws-s3') + s.add_development_dependency('aws-sdk') s.add_development_dependency('sqlite3', '~> 1.3.4') s.add_development_dependency('cucumber', '~> 1.0.0') s.add_development_dependency('aruba') diff --git a/test/storage/s3_live_test.rb b/test/storage/s3_live_test.rb index 35c2acbcc..117998d57 100644 --- a/test/storage/s3_live_test.rb +++ b/test/storage/s3_live_test.rb @@ -1,5 +1,5 @@ require './test/helper' -require 'aws/s3' +require 'aws' unless ENV["S3_TEST_BUCKET"].blank? class S3LiveTest < Test::Unit::TestCase @@ -72,7 +72,7 @@ class S3LiveTest < Test::Unit::TestCase rebuild_model :styles => { :thumb => "100x100", :square => "32x32#" }, :storage => :s3, :bucket => ENV["S3_TEST_BUCKET"], - :s3_credentials => File.new(File.join(File.dirname(__FILE__), "..", "s3.yml")) + :s3_credentials => File.new(File.join(File.dirname(__FILE__), "..", "fixtures", "s3.yml")) Dummy.delete_all @dummy = Dummy.new @@ -120,9 +120,9 @@ class S3LiveTest < Test::Unit::TestCase assert_match /.+\/question\?mark\.png/, @dummy.avatar.path end -# should "return an escaped version for url" do -# assert_match /.+\/question%3Fmark\.png/, @dummy.avatar.url -# end + should "return an escaped version for url" do + assert_match /.+\/question%3Fmark\.png/, @dummy.avatar.url + end should "be accessible" do assert_match /200 OK/, `curl -I "#{@dummy.avatar.url}"` diff --git a/test/storage/s3_test.rb b/test/storage/s3_test.rb index e371c8871..e62c5e6c7 100644 --- a/test/storage/s3_test.rb +++ b/test/storage/s3_test.rb @@ -1,5 +1,8 @@ require './test/helper' -require 'aws/s3' +require 'aws' + +AWS.stub! +AWS.config(:access_key_id => "TESTKEY", :secret_access_key => "TESTSECRET") class S3Test < Test::Unit::TestCase def rails_env(env) @@ -11,7 +14,6 @@ def rails_env(env) context "Parsing S3 credentials" do setup do @proxy_settings = {:host => "127.0.0.1", :port => 8888, :user => "foo", :password => "bar"} - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :storage => :s3, :bucket => "testing", :http_proxy => @proxy_settings, @@ -51,9 +53,73 @@ def rails_env(env) end + context "missing :bucket option" do + + setup do + rebuild_model :storage => :s3, + #:bucket => "testing", # intentionally left out + :s3_credentials => {:not => :important} + + @dummy = Dummy.new + @dummy.avatar = StringIO.new(".") + + end + + should "raise an argument error" do + exception = assert_raise(ArgumentError) { @dummy.save } + assert_match /missing required :bucket option/, exception.message + end + + end + + context ":bucket option via :s3_credentials" do + + setup do + rebuild_model :storage => :s3, :s3_credentials => {:bucket => 'testing'} + @dummy = Dummy.new + end + + should "populate #bucket_name" do + assert_equal @dummy.avatar.bucket_name, 'testing' + end + + end + + context ":bucket option" do + + setup do + rebuild_model :storage => :s3, :bucket => "testing", :s3_credentials => {} + @dummy = Dummy.new + end + + should "populate #bucket_name" do + assert_equal @dummy.avatar.bucket_name, 'testing' + end + + end + + context "missing :bucket option" do + + setup do + rebuild_model :storage => :s3, + #:bucket => "testing", # intentionally left out + :http_proxy => @proxy_settings, + :s3_credentials => {:not => :important} + + @dummy = Dummy.new + @dummy.avatar = StringIO.new(".") + + end + + should "raise an argument error" do + exception = assert_raise(ArgumentError) { @dummy.save } + assert_match /missing required :bucket option/, exception.message + end + + end + context "" do setup do - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :storage => :s3, :s3_credentials => {}, :bucket => "bucket", @@ -66,11 +132,46 @@ def rails_env(env) should "return a url based on an S3 path" do assert_match %r{^http://s3.amazonaws.com/bucket/avatars/stringio.txt}, @dummy.avatar.url end + + should "use the correct bucket" do + assert_equal "bucket", @dummy.avatar.s3_bucket.name + end + + should "use the correct key" do + assert_equal "avatars/stringio.txt", @dummy.avatar.s3_object.key + end + + end + + context "An attachment that uses S3 for storage and has the style in the path" do + setup do + rebuild_model :storage => :s3, + :bucket => "testing", + :path => ":attachment/:style/:basename.:extension", + :styles => { + :thumb => "80x80>" + }, + :s3_credentials => { + 'access_key_id' => "12345", + 'secret_access_key' => "54321" + } + + @dummy = Dummy.new + @dummy.avatar = StringIO.new(".") + @avatar = @dummy.avatar + end + + should "use an S3 object based on the correct path for the default style" do + assert_equal("avatars/original/stringio.txt", @dummy.avatar.s3_object.key) + end + + should "use an S3 object based on the correct path for the custom style" do + assert_equal("avatars/thumb/stringio.txt", @dummy.avatar.s3_object(:thumb).key) + end end context "s3_host_name" do setup do - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :storage => :s3, :s3_credentials => {}, :bucket => "bucket", @@ -83,11 +184,14 @@ def rails_env(env) should "return a url based on an :s3_host_name path" do assert_match %r{^http://s3-ap-northeast-1.amazonaws.com/bucket/avatars/stringio.txt}, @dummy.avatar.url end + + should "use the S3 bucket with the correct host name" do + assert_equal "s3-ap-northeast-1.amazonaws.com", @dummy.avatar.s3_bucket.config.s3_endpoint + end end context "An attachment that uses S3 for storage and has styles that return different file types" do setup do - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :styles => { :large => ['500x500#', :jpg] }, :storage => :s3, :bucket => "bucket", @@ -105,14 +209,21 @@ def rails_env(env) assert_match /.+\/5k.png/, @dummy.avatar.url end + should 'use the correct key for the original file mime type' do + assert_match /.+\/5k.png/, @dummy.avatar.s3_object.key + end + should "return a url containing the correct processed file mime type" do assert_match /.+\/5k.jpg/, @dummy.avatar.url(:large) end + + should "use the correct key for the processed file mime type" do + assert_match /.+\/5k.jpg/, @dummy.avatar.s3_object(:large).key + end end context "An attachment that uses S3 for storage and has spaces in file name" do setup do - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :styles => { :large => ['500x500#', :jpg] }, :storage => :s3, :bucket => "bucket", @@ -136,7 +247,6 @@ def rails_env(env) context "" do setup do - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :storage => :s3, :s3_credentials => {}, :bucket => "bucket", @@ -153,7 +263,6 @@ def rails_env(env) context "" do setup do - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :storage => :s3, :s3_credentials => { :production => { :bucket => "prod_bucket" }, @@ -173,7 +282,6 @@ def rails_env(env) context "generating a url with a proc as the host alias" do setup do - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :storage => :s3, :s3_credentials => { :bucket => "prod_bucket" }, :s3_host_alias => Proc.new{|atch| "cdn#{atch.instance.counter % 4}.example.com"}, @@ -203,7 +311,6 @@ def counter context "" do setup do - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :storage => :s3, :s3_credentials => {}, :bucket => "bucket", @@ -220,7 +327,6 @@ def counter context "Generating a secure url with an expiration" do setup do - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :storage => :s3, :s3_credentials => { :production => { :bucket => "prod_bucket" }, @@ -236,7 +342,9 @@ def counter @dummy = Dummy.new @dummy.avatar = StringIO.new(".") - AWS::S3::S3Object.expects(:url_for).with("avatars/stringio.txt", "prod_bucket", { :expires_in => 3600, :use_ssl => true }) + object = stub + @dummy.avatar.stubs(:s3_object).returns(object) + object.expects(:url_for).with(:read, :expires => 3600, :secure => true) @dummy.avatar.expiring_url end @@ -246,9 +354,8 @@ def counter end end - context "Generating a url with an expiration" do + context "Generating a url with an expiration for each style" do setup do - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :storage => :s3, :s3_credentials => { :production => { :bucket => "prod_bucket" }, @@ -263,22 +370,25 @@ def counter @dummy = Dummy.new @dummy.avatar = StringIO.new(".") + end - AWS::S3::S3Object.expects(:url_for).with("avatars/original/stringio.txt", "prod_bucket", { :expires_in => 3600, :use_ssl => true }) - @dummy.avatar.expiring_url - - AWS::S3::S3Object.expects(:url_for).with("avatars/thumb/stringio.txt", "prod_bucket", { :expires_in => 1800, :use_ssl => true }) + should "should generate a url for the thumb" do + object = stub + @dummy.avatar.stubs(:s3_object).with(:thumb).returns(object) + object.expects(:url_for).with(:read, :expires => 1800, :secure => true) @dummy.avatar.expiring_url(1800, :thumb) end - should "should succeed" do - assert true + should "should generate a url for the default style" do + object = stub + @dummy.avatar.stubs(:s3_object).with(:original).returns(object) + object.expects(:url_for).with(:read, :expires => 1800, :secure => true) + @dummy.avatar.expiring_url(1800) end end context "Parsing S3 credentials with a bucket in them" do setup do - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :storage => :s3, :s3_credentials => { :production => { :bucket => "prod_bucket" }, @@ -290,18 +400,20 @@ def counter should "get the right bucket in production" do rails_env("production") assert_equal "prod_bucket", @dummy.avatar.bucket_name + assert_equal "prod_bucket", @dummy.avatar.s3_bucket.name end should "get the right bucket in development" do rails_env("development") assert_equal "dev_bucket", @dummy.avatar.bucket_name + assert_equal "dev_bucket", @dummy.avatar.s3_bucket.name end end context "Parsing S3 credentials with a s3_host_name in them" do setup do - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :storage => :s3, + :bucket => 'testing', :s3_credentials => { :production => { :s3_host_name => "s3-world-end.amazonaws.com" }, :development => { :s3_host_name => "s3-ap-northeast-1.amazonaws.com" } @@ -312,16 +424,19 @@ def counter should "get the right s3_host_name in production" do rails_env("production") assert_match %r{^s3-world-end.amazonaws.com}, @dummy.avatar.s3_host_name + assert_match %r{^s3-world-end.amazonaws.com}, @dummy.avatar.s3_bucket.config.s3_endpoint end should "get the right s3_host_name in development" do rails_env("development") assert_match %r{^s3-ap-northeast-1.amazonaws.com}, @dummy.avatar.s3_host_name + assert_match %r{^s3-ap-northeast-1.amazonaws.com}, @dummy.avatar.s3_bucket.config.s3_endpoint end should "get the right s3_host_name if the key does not exist" do rails_env("test") assert_match %r{^s3.amazonaws.com}, @dummy.avatar.s3_host_name + assert_match %r{^s3.amazonaws.com}, @dummy.avatar.s3_bucket.config.s3_endpoint end end @@ -361,7 +476,11 @@ def counter context "and saved" do setup do - AWS::S3::S3Object.stubs(:store).with(@dummy.avatar.path, anything, 'testing', :content_type => 'image/png', :access => :public_read) + object = stub + @dummy.avatar.stubs(:s3_object).returns(object) + object.expects(:write).with(anything, + :content_type => "image/png", + :acl => :public_read) @dummy.save end @@ -371,7 +490,6 @@ def counter end should "delete tempfiles" do - AWS::S3::S3Object.stubs(:store).with(@dummy.avatar.path, anything, 'testing', :content_type => 'image/png', :access => :public_read) File.stubs(:exist?).returns(true) Paperclip::Tempfile.any_instance.expects(:close).at_least_once() Paperclip::Tempfile.any_instance.expects(:unlink).at_least_once() @@ -381,11 +499,12 @@ def counter context "and saved without a bucket" do setup do - class AWS::S3::NoSuchBucket < AWS::S3::ResponseError - # Force the class to be created as a proper subclass of ResponseError thanks to AWS::S3's autocreation of exceptions - end - AWS::S3::Bucket.expects(:create).with("testing") - AWS::S3::S3Object.stubs(:store).raises(AWS::S3::NoSuchBucket.new(:message, :response)).then.returns(true) + AWS::S3::BucketCollection.any_instance.expects(:create).with("testing") + AWS::S3::S3Object.any_instance.stubs(:write). + raises(AWS::S3::Errors::NoSuchBucket.new(stub, + stub(:status => 404, + :body => ""))). + then.returns(nil) @dummy.save end @@ -396,8 +515,8 @@ class AWS::S3::NoSuchBucket < AWS::S3::ResponseError context "and remove" do setup do - AWS::S3::S3Object.stubs(:exists?).returns(true) - AWS::S3::S3Object.stubs(:delete) + AWS::S3::S3Object.any_instance.stubs(:exists?).returns(true) + AWS::S3::S3Object.any_instance.stubs(:delete) @dummy.destroy_attached_files end @@ -410,7 +529,6 @@ class AWS::S3::NoSuchBucket < AWS::S3::ResponseError context "An attachment with S3 storage and bucket defined as a Proc" do setup do - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :storage => :s3, :bucket => lambda { |attachment| "bucket_#{attachment.instance.other}" }, :s3_credentials => {:not => :important} @@ -418,13 +536,14 @@ class AWS::S3::NoSuchBucket < AWS::S3::ResponseError should "get the right bucket name" do assert "bucket_a", Dummy.new(:other => 'a').avatar.bucket_name + assert "bucket_a", Dummy.new(:other => 'a').avatar.s3_bucket.name assert "bucket_b", Dummy.new(:other => 'b').avatar.bucket_name + assert "bucket_b", Dummy.new(:other => 'b').avatar.s3_bucket.name end end context "An attachment with S3 storage and specific s3 headers set" do setup do - AWS::S3::Base.stubs(:establish_connection!) rebuild_model :storage => :s3, :bucket => "testing", :path => ":attachment/:style/:basename.:extension", @@ -446,13 +565,168 @@ class AWS::S3::NoSuchBucket < AWS::S3::ResponseError context "and saved" do setup do - AWS::S3::Base.stubs(:establish_connection!) - AWS::S3::S3Object.stubs(:store).with(@dummy.avatar.path, - anything, - 'testing', - :content_type => 'image/png', - :access => :public_read, - 'Cache-Control' => 'max-age=31557600') + object = stub + @dummy.avatar.stubs(:s3_object).returns(object) + object.expects(:write).with(anything, + :content_type => "image/png", + :acl => :public_read, + :cache_control => 'max-age=31557600') + @dummy.save + end + + should "succeed" do + assert true + end + end + end + end + + context "An attachment with S3 storage and metadata set using header names" do + setup do + rebuild_model :storage => :s3, + :bucket => "testing", + :path => ":attachment/:style/:basename.:extension", + :s3_credentials => { + 'access_key_id' => "12345", + 'secret_access_key' => "54321" + }, + :s3_headers => {'x-amz-meta-color' => 'red'} + end + + context "when assigned" do + setup do + @file = File.new(File.join(File.dirname(__FILE__), '..', 'fixtures', '5k.png'), 'rb') + @dummy = Dummy.new + @dummy.avatar = @file + end + + teardown { @file.close } + + context "and saved" do + setup do + object = stub + @dummy.avatar.stubs(:s3_object).returns(object) + object.expects(:write).with(anything, + :content_type => "image/png", + :acl => :public_read, + :metadata => { "color" => "red" }) + @dummy.save + end + + should "succeed" do + assert true + end + end + end + end + + context "An attachment with S3 storage and metadata set using the :s3_metadata option" do + setup do + rebuild_model :storage => :s3, + :bucket => "testing", + :path => ":attachment/:style/:basename.:extension", + :s3_credentials => { + 'access_key_id' => "12345", + 'secret_access_key' => "54321" + }, + :s3_metadata => { "color" => "red" } + end + + context "when assigned" do + setup do + @file = File.new(File.join(File.dirname(__FILE__), '..', 'fixtures', '5k.png'), 'rb') + @dummy = Dummy.new + @dummy.avatar = @file + end + + teardown { @file.close } + + context "and saved" do + setup do + object = stub + @dummy.avatar.stubs(:s3_object).returns(object) + object.expects(:write).with(anything, + :content_type => "image/png", + :acl => :public_read, + :metadata => { "color" => "red" }) + @dummy.save + end + + should "succeed" do + assert true + end + end + end + end + + context "An attachment with S3 storage and storage class set using the header name" do + setup do + rebuild_model :storage => :s3, + :bucket => "testing", + :path => ":attachment/:style/:basename.:extension", + :s3_credentials => { + 'access_key_id' => "12345", + 'secret_access_key' => "54321" + }, + :s3_headers => { "x-amz-storage-class" => "reduced_redundancy" } + end + + context "when assigned" do + setup do + @file = File.new(File.join(File.dirname(__FILE__), '..', 'fixtures', '5k.png'), 'rb') + @dummy = Dummy.new + @dummy.avatar = @file + end + + teardown { @file.close } + + context "and saved" do + setup do + object = stub + @dummy.avatar.stubs(:s3_object).returns(object) + object.expects(:write).with(anything, + :content_type => "image/png", + :acl => :public_read, + :storage_class => "reduced_redundancy") + @dummy.save + end + + should "succeed" do + assert true + end + end + end + end + + context "An attachment with S3 storage and storage class set using the :storage_class option" do + setup do + rebuild_model :storage => :s3, + :bucket => "testing", + :path => ":attachment/:style/:basename.:extension", + :s3_credentials => { + 'access_key_id' => "12345", + 'secret_access_key' => "54321" + }, + :s3_storage_class => :reduced_redundancy + end + + context "when assigned" do + setup do + @file = File.new(File.join(File.dirname(__FILE__), '..', 'fixtures', '5k.png'), 'rb') + @dummy = Dummy.new + @dummy.avatar = @file + end + + teardown { @file.close } + + context "and saved" do + setup do + object = stub + @dummy.avatar.stubs(:s3_object).returns(object) + object.expects(:write).with(anything, + :content_type => "image/png", + :acl => :public_read, + :storage_class => :reduced_redundancy) @dummy.save end @@ -476,13 +750,12 @@ class AWS::S3::NoSuchBucket < AWS::S3::ResponseError Dummy.delete_all @dummy = Dummy.new - @dummy.avatar.send(:establish_connection!) end should "parse the credentials" do assert_equal 'pathname_bucket', @dummy.avatar.bucket_name - assert_equal 'pathname_key', AWS::S3::Base.connection.options[:access_key_id] - assert_equal 'pathname_secret', AWS::S3::Base.connection.options[:secret_access_key] + assert_equal 'pathname_key', @dummy.avatar.s3_bucket.config.access_key_id + assert_equal 'pathname_secret', @dummy.avatar.s3_bucket.config.secret_access_key end end @@ -500,13 +773,12 @@ class AWS::S3::NoSuchBucket < AWS::S3::ResponseError Dummy.delete_all @dummy = Dummy.new - @dummy.avatar.send(:establish_connection!) end should "run the file through ERB" do assert_equal 'env_bucket', @dummy.avatar.bucket_name - assert_equal 'env_key', AWS::S3::Base.connection.options[:access_key_id] - assert_equal 'env_secret', AWS::S3::Base.connection.options[:secret_access_key] + assert_equal 'env_key', @dummy.avatar.s3_bucket.config.access_key_id + assert_equal 'env_secret', @dummy.avatar.s3_bucket.config.secret_access_key end end @@ -533,12 +805,11 @@ class AWS::S3::NoSuchBucket < AWS::S3::ResponseError context "and saved" do setup do - AWS::S3::Base.stubs(:establish_connection!) - AWS::S3::S3Object.expects(:store).with(@dummy.avatar.path, - anything, - 'testing', - :content_type => 'image/png', - :access => :public_read) + object = stub + @dummy.avatar.stubs(:s3_object).returns(object) + object.expects(:write).with(anything, + :content_type => "image/png", + :acl => :public_read) @dummy.save end @@ -572,12 +843,11 @@ class AWS::S3::NoSuchBucket < AWS::S3::ResponseError context "and saved" do setup do - AWS::S3::Base.stubs(:establish_connection!) - AWS::S3::S3Object.expects(:store).with(@dummy.avatar.path, - anything, - 'testing', - :content_type => 'image/png', - :access => :private) + object = stub + @dummy.avatar.stubs(:s3_object).returns(object) + object.expects(:write).with(anything, + :content_type => "image/png", + :acl => :private) @dummy.save end @@ -617,13 +887,12 @@ class AWS::S3::NoSuchBucket < AWS::S3::ResponseError context "and saved" do setup do - AWS::S3::Base.stubs(:establish_connection!) [:thumb, :original].each do |style| - AWS::S3::S3Object.expects(:store).with("avatars/#{style}/5k.png", - anything, - 'testing', - :content_type => 'image/png', - :access => style == :thumb ? :public_read : :private) + object = stub + @dummy.avatar.stubs(:s3_object).with(style).returns(object) + object.expects(:write).with(anything, + :content_type => "image/png", + :acl => style == :thumb ? :public_read : :private) end @dummy.save end @@ -666,15 +935,12 @@ class AWS::S3::NoSuchBucket < AWS::S3::ResponseError context "and saved" do setup do - AWS::S3::Base.stubs(:establish_connection!) [:thumb, :original].each do |style| - AWS::S3::S3Object.expects(:store).with( - "avatars/#{style}/5k.png", - anything, - 'testing', - :content_type => 'image/png', - :access => style == :thumb ? :public_read : :private - ) + object = stub + @dummy.avatar.stubs(:s3_object).with(style).returns(object) + object.expects(:write).with(anything, + :content_type => "image/png", + :acl => style == :thumb ? :public_read : :private) end @dummy.save end