test/backends/remote/s3_test.rb @@ -1,14 +1,14 @@ development: - secret_access_key: AbCDEfGHiJKlmNOPQRS1 - access_key_id: 1234567891abcdeFGHI/JKL+MnoPQrsT123UvwX4 - bucket_prefix: appname_development + bucket_name: appname_development + access_key_id: + secret_access_key: test: - secret_access_key: AbCDEfGHiJKlmNOPQRS1 - access_key_id: 1234567891abcdeFGHI/JKL+MnoPQrsT123UvwX4 - bucket_prefix: appname_test + bucket_name: appname_test + access_key_id: + secret_access_key: production: - secret_access_key: AbCDEfGHiJKlmNOPQRS1 - access_key_id: 1234567891abcdeFGHI/JKL+MnoPQrsT123UvwX4 - bucket_prefix: appname \ No newline at end of file + bucket_name: appname + access_key_id: + secret_access_key: amazon_s3.yml.tpl @@ -18,8 +18,8 @@ module Technoweenie # :nodoc: # * <tt>:resize_to</tt> - Used by RMagick to resize images. Pass either an array of width/height, or a geometry string. # * <tt>:thumbnails</tt> - Specifies a set of thumbnails to generate. This accepts a hash of filename suffixes and RMagick resizing options. # * <tt>:thumbnail_class</tt> - Set what class to use for thumbnails. This attachment class is used by default. - # * <tt>:file_system_path</tt> - path to store the uploaded files. Uses public/#{table_name} by default. - # Setting this sets the :storage to :file_system. + # * <tt>:path_prefix</tt> - path to store the uploaded files. Uses public/#{table_name} by default for the filesystem, and just #{table_name} + # for the S3 backend. Setting this sets the :storage to :file_system. # * <tt>:storage</tt> - Use :file_system to specify the attachment data is stored with the file system. Defaults to :db_system. # # Examples: @@ -30,10 +30,10 @@ module Technoweenie # :nodoc: # has_attachment :content_type => :image, :resize_to => [50,50] # has_attachment :content_type => ['application/pdf', :image], :resize_to => 'x50' # has_attachment :thumbnails => { :thumb => [50, 50], :geometry => 'x50' } - # has_attachment :storage => :file_system, :file_system_path => 'public/files' - # has_attachment :storage => :file_system, :file_system_path => 'public/files', + # has_attachment :storage => :file_system, :path_prefix => 'public/files' + # has_attachment :storage => :file_system, :path_prefix => 'public/files', # :content_type => :image, :resize_to => [50,50] - # has_attachment :storage => :file_system, :file_system_path => 'public/files', + # has_attachment :storage => :file_system, :path_prefix => 'public/files', # :thumbnails => { :thumb => [50, 50], :geometry => 'x50' } # has_attachment :storage => :s3 def has_attachment(options = {}) @@ -50,9 +50,12 @@ module Technoweenie # :nodoc: class_inheritable_accessor :attachment_options attr_accessor :thumbnail_resize_options - options[:storage] ||= options[:file_system_path] ? :file_system : :db_file - options[:file_system_path] ||= File.join("public", table_name) - options[:file_system_path] = options[:file_system_path][1..-1] if options[:file_system_path].first == '/' + options[:storage] ||= (options[:file_system_path] || options[:path_prefix]) ? :file_system : :db_file + options[:path_prefix] ||= options[:file_system_path] + if options[:path_prefix].nil? + options[:path_prefix] = options[:storage] == :s3 ? table_name : File.join("public", table_name) + end + options[:path_prefix] = options[:path_prefix][1..-1] if options[:path_prefix].first == '/' with_options :foreign_key => 'parent_id' do |m| m.has_many :thumbnails, :dependent => :destroy, :class_name => options[:thumbnail_class].to_s lib/technoweenie/attachment_fu.rb @@ -16,7 +16,7 @@ module Technoweenie # :nodoc: # Overwrite this method in your model to customize the filename. # The optional thumbnail argument will output the thumbnail's filename. def full_filename(thumbnail = nil) - file_system_path = (thumbnail ? thumbnail_class : self).attachment_options[:file_system_path].to_s + file_system_path = (thumbnail ? thumbnail_class : self).attachment_options[:path_prefix].to_s File.join(RAILS_ROOT, file_system_path, attachment_path_id, thumbnail_name_for(thumbnail)) end lib/technoweenie/attachment_fu/backends/file_system.rb @@ -3,50 +3,58 @@ module Technoweenie # :nodoc: module Backends # = AWS::S3 Storage Backend # - # Enables use of Amazon's Simple Storage Service (http://aws.amazon.com/s3) as a storage mechanism + # Enables use of {Amazon's Simple Storage Service}[http://aws.amazon.com/s3] as a storage mechanism # # == Requirements # - # Requires the AWS::S3 Library for S3 by Marcel Molina Jr. (http://amazon.rubyforge.org) installed either + # Requires the {AWS::S3 Library}[http://amazon.rubyforge.org] for S3 by Marcel Molina Jr. installed either # as a gem or a as a Rails plugin. # # == Configuration # # Configuration is done via <tt>RAILS_ROOT/config/amazon_s3.yml</tt> and is loaded according to the <tt>RAILS_ENV</tt>. - # The minimum connection options that you must specify are your access key id and your secret access key. + # The minimum connection options that you must specify are a bucket name, your access key id and your secret access key. # If you don't already have your access keys, all you need to sign up for the S3 service is an account at Amazon. # You can sign up for S3 and get access keys by visiting http://aws.amazon.com/s3. - # + # # Example configuration (RAILS_ROOT/config/amazon_s3.yml) # # development: - # secret_access_key: AbCDEfGHiJKlmNOPQRS1 - # access_key_id: 1234567891abcdeFGHI/JKL+MnoPQrsT123UvwX4 - # bucket_prefix: appname_development + # bucket_name: appname_development + # access_key_id: <your key> + # secret_access_key: <your key> # # test: - # secret_access_key: AbCDEfGHiJKlmNOPQRS1 - # access_key_id: 1234567891abcdeFGHI/JKL+MnoPQrsT123UvwX4 - # bucket_prefix: appname_test + # bucket_name: appname_test + # access_key_id: <your key> + # secret_access_key: <your key> # # production: - # secret_access_key: AbCDEfGHiJKlmNOPQRS1 - # access_key_id: 1234567891abcdeFGHI/JKL+MnoPQrsT123UvwX4 - # bucket_prefix: appname + # bucket_name: appname + # access_key_id: <your key> + # secret_access_key: <your key> # - # === Required arguments + # === Required configuration parameters # # * <tt>:access_key_id</tt> - The access key id for your S3 account. Provided by Amazon. # * <tt>:secret_access_key</tt> - The secret access key for your S3 account. Provided by Amazon. - # * <tt>:bucket_prefix</tt> - The string prefix to assign to each bucket. Used to create unique bucket names in the format <tt>#{bucket_prefix}_#{table_name}</tt>. + # * <tt>:bucket_name</tt> - A unique bucket name (think of the bucket_name as being like a database name). # # If any of these required arguments is missing, a MissingAccessKey exception will be raised from AWS::S3. # - # === Optional arguments + # == About bucket names + # + # Bucket names have to be globaly unique across the S3 system. And you can only have up to 100 of them, + # so it's a good idea to think of a bucket as being like a database, hence the correspondance in this + # implementation to the development, test, and production environments. + # + # The number of objects you can store in a bucket is, for all intents and purposes, unlimited. # - # * <tt>:server</tt> - The server to make requests to. You can use this to specify your bucket in the subdomain, or your own domain's cname if you are using virtual hosted buckets. Defaults to <tt>s3.amazonaws.com</tt>. + # === Optional configuration parameters + # + # * <tt>:server</tt> - The server to make requests to. Defaults to <tt>s3.amazonaws.com</tt>. # * <tt>:port</tt> - The port to the requests should be made on. Defaults to 80 or 443 if <tt>:use_ssl</tt> is set. - # * <tt>:use_ssl</tt> - Whether requests should be made over SSL. If set to true, <tt>:port</tt> will be implicitly set to 443, unless specified otherwise. Defaults to false. + # * <tt>:use_ssl</tt> - If set to true, <tt>:port</tt> will be implicitly set to 443, unless specified otherwise. Defaults to false. # # == Usage # @@ -56,79 +64,200 @@ module Technoweenie # :nodoc: # has_attachment :storage => :s3 # end # - # Of course, all the usual configuration options apply: + # === Customizing the path + # + # By default, files are prefixed using a pseudo hierarchy in the form of <tt>:table_name/:id</tt>, which results + # in S3 urls that look like: http(s)://:server/:bucket_name/:table_name/:id/:filename with :table_name + # representing the customizable portion of the path. You can customize this prefix using the <tt>:path_prefix</tt> + # option: + # + # class Photo < ActiveRecord::Base + # has_attachment :storage => :s3, :path_prefix => 'my/custom/path' + # end + # + # Which would result in URLs like <tt>http(s)://:server/:bucket_name/my/custom/path/:id/:filename.</tt> + # + # === Permissions + # + # By default, files are stored on S3 with public access permissions. You can customize this using + # the <tt>:s3_access</tt> option to <tt>has_attachment</tt>. Available values are + # <tt>:private</tt>, <tt>:public_read_write</tt>, and <tt>:authenticated_read</tt>. + # + # === Other options + # + # Of course, all the usual configuration options apply, such as content_type and thumbnails: # - # has_attachment :storage => :s3, :content_type => ['application/pdf', :image], :resize_to => 'x50' - # has_attachment :storage => :s3, :thumbnails => { :thumb => [50, 50], :geometry => 'x50' } + # class Photo < ActiveRecord::Base + # has_attachment :storage => :s3, :content_type => ['application/pdf', :image], :resize_to => 'x50' + # has_attachment :storage => :s3, :thumbnails => { :thumb => [50, 50], :geometry => 'x50' } + # end + # + # === Accessing S3 URLs + # + # You can get an object's URL using the s3_url accessor. For example, assuming that for your postcard app + # you had a bucket name like 'postcard_world_development', and an attachment model called Photo: + # + # @postcard.s3_url # => http(s)://s3.amazonaws.com/postcard_world_development/photos/1/mexico.jpg + # + # The resulting url is in the form: http(s)://:server/:bucket_name/:table_name/:id/:file. + # The optional thumbnail argument will output the thumbnail's filename (if any). + # + # Additionally, you can get an object's base path relative to the bucket root using + # <tt>base_path</tt>: + # + # @photo.file_base_path # => photos/1 + # + # And the full path (including the filename) using <tt>full_filename</tt>: + # + # @photo.full_filename # => photos/ + # + # Niether <tt>base_path</tt> or <tt>full_filename</tt> include the bucket name as part of the path. + # You can retrieve the bucket name using the <tt>bucket_name</tt> method. module S3 - class S3RequiredLibraryNotFound < StandardError; end - class S3ConfigFileNotFound < StandardError; end - class S3BucketExists < StandardError; end + class S3RequiredLibraryNotFoundError < StandardError; end + class S3ConfigFileNotFoundError < StandardError; end def self.included(base) #:nodoc: + mattr_reader :bucket_name, :s3_config + begin require 'aws/s3' + include AWS::S3 rescue LoadError - raise S3RequiredLibraryNotFound.new('AWS::S3 could not be loaded. Try installing with sudo gem i aws-s3, or see http://amazon.rubyforge.org for more information') + raise S3RequiredLibraryNotFoundError.new('AWS::S3 could not be loaded') end begin @@s3_config = YAML.load_file(RAILS_ROOT + '/config/amazon_s3.yml')[ENV['RAILS_ENV']].symbolize_keys rescue - raise S3ConfigFileNotFound.new('File RAILS_ROOT/config/amazon_s3.yml not found') + raise S3ConfigFileNotFoundError.new('File RAILS_ROOT/config/amazon_s3.yml not found') end - @@bucket = [@@s3_config.delete(:bucket_prefix), base.table_name].join('_') - mattr_reader :s3_config, :bucket + @@bucket_name = s3_config[:bucket_name] + + Base.establish_connection!( + :access_key_id => s3_config[:access_key_id], + :secret_access_key => s3_config[:secret_access_key], + :server => s3_config[:server], + :port => s3_config[:port], + :use_ssl => s3_config[:use_ssl] + ) - AWS::S3::Base.establish_connection!(s3_config) - find_or_create_bucket(bucket) + # Bucket.create(@@bucket_name) base.before_update :rename_file end - - def self.find_or_create_bucket(name) - AWS::S3::Bucket.find(name) - rescue AWS::S3::NoSuchBucket - AWS::S3::Bucket.create(name) - rescue AWS::S3::AccessDenied - raise S3BucketExists.new("Bucket name already exists: #{name}. Use a different bucket_prefix in RAILS_ROOT/config/amazon_s3.yml") + + # Overwrites the base filename writer in order to store the old filename + def filename=(value) + @old_filename = filename unless filename.nil? || @old_filename + write_attribute :filename, sanitize_filename(value) + end + + # The attachment ID used in the full path of a file + def attachment_path_id + ((respond_to?(:parent_id) && parent_id) || id).to_s + end + + # The pseudo hierarchy containing the file relative to the bucket name + # Example: <tt>:table_name/:id</tt> + def base_path + File.join(attachment_options[:path_prefix], attachment_path_id) + end + + # The full path to the file relative to the bucket name + # Example: <tt>:table_name/:id/:filename</tt> + def full_filename(thumbnail = nil) + File.join(base_path, thumbnail_name_for(thumbnail)) end - # Generates an S3 URL for the file in the form of: http(s)://<tt>{server}</tt>/<tt>{bucket_name}</tt>/<tt>{file_name}</tt> - # The <tt>{server}</tt> variable defaults to <tt>AWS::S3 URL::DEFAULT_HOST</tt> (http://s3.amazonaws.com) and can be - # set using the configuration parameters in <tt>RAILS_ROOT/config/amazon_s3.yml</tt> + # All public objects are accessible via a GET request to the S3 servers. You can generate a + # url for an object using the s3_url method. # - # Example usage: <tt>image_tag(@photo.s3_url)</tt> + # @photo.s3_url + # + # The resulting url is in the form: <tt>http(s)://:server/:bucket_name/:table_name/:id/:file</tt> where + # the <tt>:server</tt> variable defaults to <tt>AWS::S3 URL::DEFAULT_HOST</tt> (s3.amazonaws.com) and can be + # set using the configuration parameters in <tt>RAILS_ROOT/config/amazon_s3.yml</tt>. + # + # The optional thumbnail argument will output the thumbnail's filename (if any). def s3_url(thumbnail = nil) - s3_config[:use_ssl] ? 'https://' : 'http://' + (s3_config[:server] || AWS::S3::DEFAULT_HOST) + '/' + bucket + '/' + thumbnail_name_for(thumbnail) + protocol, hostname = s3_config[:use_ssl] ? 'https://' : 'http://', s3_config[:server] || DEFAULT_HOST + protocol + File.join(hostname, bucket_name, full_filename(thumbnail)) end alias :public_filename :s3_url + # All private objects are accessible via an authenticated GET request to the S3 servers. You can generate an + # authenticated url for an object like this: + # + # @photo.authenticated_s3_url + # + # By default authenticated urls expire 5 minutes after they were generated. + # + # Expiration options can be specified either with an absolute time using the <tt>:expires</tt> option, + # or with a number of seconds relative to now with the <tt>:expires_in</tt> option: + # + # # Absolute expiration date (October 13th, 2025) + # @photo.authenticated_s3_url(:expires => Time.mktime(2025,10,13).to_i) + # + # # Expiration in five hours from now + # @photo.authenticated_s3_url(:expires_in => 5.hours) + # + # You can specify whether the url should go over SSL with the <tt>:use_ssl</tt> option. + # By default, the ssl settings for the current connection will be used: + # + # @photo.authenticated_s3_url(:use_ssl => true) + # + # Finally, the optional thumbnail argument will output the thumbnail's filename (if any): + # + # @photo.authenticated_s3_url('thumbnail', :expires_in => 5.hours, :use_ssl => true) + def authenticated_s3_url(*args) + thumbnail = args.first.is_a?(String) ? args.first : nil + options = args.last.is_a?(Hash) ? args.last : {} + S3Object.url_for(full_filename(thumbnail), bucket_name, options) + end + def create_temp_file write_to_temp_file current_data end def current_data - AWS::S3::S3Object.value filename, bucket + S3Object.value full_filename, bucket_name end protected - # Destroys the file. Called in the after_destroy callback + # Called in the after_destroy callback def destroy_file - AWS::S3::S3Object.delete filename, bucket + S3Object.delete full_filename, bucket_name end - + def rename_file return unless @old_filename && @old_filename != filename - AWS::S3::S3Object.rename(@old_filename, filename, bucket, :access => attachment_options[:s3_access]) + + old_full_filename = File.join(base_path, @old_filename) + + S3Object.rename( + old_full_filename, + full_filename, + bucket_name, + :access => attachment_options[:s3_access] + ) + @old_filename = nil true end - - # Saves the file to S3 + def save_to_storage - AWS::S3::S3Object.store(filename, temp_data, bucket, :content_type => content_type, :access => attachment_options[:s3_access]) if save_attachment? + if save_attachment? + S3Object.store( + full_filename, + temp_data, + bucket_name, + :content_type => content_type, + :access => attachment_options[:s3_access] + ) + end + @old_filename = nil true end lib/technoweenie/attachment_fu/backends/s3.rb @@ -40,17 +40,17 @@ class ImageWithThumbsAttachment < Attachment end class FileAttachment < ActiveRecord::Base - has_attachment :file_system_path => 'vendor/plugins/attachment_fu/test/files', :processor => :rmagick + has_attachment :path_prefix => 'vendor/plugins/attachment_fu/test/files', :processor => :rmagick validates_as_attachment end class ImageFileAttachment < FileAttachment - has_attachment :file_system_path => 'vendor/plugins/attachment_fu/test/files', + has_attachment :path_prefix => 'vendor/plugins/attachment_fu/test/files', :content_type => :image, :resize_to => [50,50] end class ImageWithThumbsFileAttachment < FileAttachment - has_attachment :file_system_path => 'vendor/plugins/attachment_fu/test/files', + has_attachment :path_prefix => 'vendor/plugins/attachment_fu/test/files', :thumbnails => { :thumb => [50, 50], :geometry => 'x50' }, :resize_to => [55,55] after_resize do |record, img| record.aspect_ratio = img.columns.to_f / img.rows.to_f @@ -58,13 +58,14 @@ class ImageWithThumbsFileAttachment < FileAttachment end class ImageWithThumbsClassFileAttachment < FileAttachment + # use file_system_path to test backwards compatibility has_attachment :file_system_path => 'vendor/plugins/attachment_fu/test/files', :thumbnails => { :thumb => [50, 50] }, :resize_to => [55,55], :thumbnail_class => 'ImageThumbnail' end class ImageThumbnail < FileAttachment - has_attachment :file_system_path => 'vendor/plugins/attachment_fu/test/files/thumbnails' + has_attachment :path_prefix => 'vendor/plugins/attachment_fu/test/files/thumbnails' end # no parent @@ -75,7 +76,7 @@ end # no filename, no size, no content_type class MinimalAttachment < ActiveRecord::Base - has_attachment :file_system_path => 'vendor/plugins/attachment_fu/test/files', :processor => :rmagick + has_attachment :path_prefix => 'vendor/plugins/attachment_fu/test/files', :processor => :rmagick validates_as_attachment def filename @@ -85,8 +86,21 @@ end begin class ImageScienceAttachment < ActiveRecord::Base - has_attachment :file_system_path => 'vendor/plugins/attachment_fu/test/files', + has_attachment :path_prefix => 'vendor/plugins/attachment_fu/test/files', :processor => :image_science, :thumbnails => { :thumb => [50, 51], :geometry => '31>' }, :resize_to => 55 end rescue MissingSourceFile -end \ No newline at end of file +end + +begin + class S3Attachment < ActiveRecord::Base + has_attachment :storage => :s3, :processor => :rmagick + validates_as_attachment + end + + class S3WithPathPrefixAttachment < S3Attachment + has_attachment :storage => :s3, :path_prefix => 'some/custom/path/prefix', :processor => :rmagick + validates_as_attachment + end +rescue Technoweenie::AttachmentFu::Backends::S3::S3ConfigFileNotFoundError +end test/fixtures/attachment.rb @@ -49,4 +49,16 @@ ActiveRecord::Schema.define(:version => 0) do create_table :db_files, :force => true do |t| t.column :data, :binary end + + create_table :s3_attachments, :force => true do |t| + t.column :parent_id, :integer + t.column :thumbnail, :string + t.column :filename, :string, :limit => 255 + t.column :content_type, :string, :limit => 255 + t.column :size, :integer + t.column :width, :integer + t.column :height, :integer + t.column :type, :string + t.column :aspect_ratio, :float + end end \ No newline at end of file test/schema.rb @@ -1,5 +1,7 @@ $:.unshift(File.dirname(__FILE__) + '/../lib') +ENV['RAILS_ENV'] = 'test' + require 'test/unit' require File.expand_path(File.join(File.dirname(__FILE__), '../../../../config/environment.rb')) require 'breakpoint' test/test_helper.rb 4b76298835008a22702e544b7db19df343601e66 technoweenie technoweenie@567b1171-46fb-0310-a4c9-b4bef9110e78 http://github.com/crafterm/attachment_fu/commit/7be8acf8c42445b5e2d7306989d6731fe8ce4b42 7be8acf8c42445b5e2d7306989d6731fe8ce4b42 2007-01-14T10:35:00-08:00 2007-01-14T10:35:00-08:00 updates to s3 support with remote tests git-svn-id: http://svn.techno-weenie.net/projects/plugins/attachment_fu@2675 567b1171-46fb-0310-a4c9-b4bef9110e78 8e88ae9a305c3316e8d6a7ad011d796fe054d9ef technoweenie technoweenie@567b1171-46fb-0310-a4c9-b4bef9110e78