lib/astrotrain/imap.rb spec/fixtures/apple_multipart.txt spec/fixtures/fwd.txt spec/fixtures/iso-8859-1.txt spec/fixtures/multipart.txt spec/fixtures/multiple_delivered_to.txt spec/fixtures/multiple_with_body_recipients.txt spec/fixtures/utf-8.txt spec/models/astrotrain/imap_spec.rb vendor/rest-client/README.rdoc vendor/rest-client/Rakefile vendor/rest-client/bin/restclient vendor/rest-client/lib/rest_client.rb vendor/rest-client/lib/rest_client/net_http_ext.rb vendor/rest-client/lib/rest_client/payload.rb vendor/rest-client/lib/rest_client/request_errors.rb vendor/rest-client/lib/rest_client/resource.rb vendor/rest-client/rest-client.gemspec vendor/rest-client/spec/base.rb vendor/rest-client/spec/master_shake.jpg vendor/rest-client/spec/payload_spec.rb vendor/rest-client/spec/request_errors_spec.rb vendor/rest-client/spec/resource_spec.rb vendor/rest-client/spec/rest_client_spec.rb @@ -1,4 +1,4 @@ -class Exceptions < Application +class Exceptions < Merb::Controller # handle NotFound exceptions (404) def not_found app/controllers/exceptions.rb @@ -1,16 +1,27 @@ class LoggedMail include DataMapper::Resource - property :id, Serial - property :mapping_id, Integer, :nullable => false, :index => true - property :recipient, String - property :subject, String - property :raw, Text - property :created_at, DateTime + class << self + attr_accessor :log_path + end + + self.log_path = Merb.root / 'messages' + + property :id, Serial + property :mapping_id, Integer, :index => true + property :recipient, String + property :subject, String + property :created_at, DateTime + property :delivered_at, DateTime + property :error_message, String belongs_to :mapping def self.from(message) - new :recipient => message.recipient, :subject => message.subject, :raw => message.raw + logged = new(:subject => message.subject) + if !block_given? || yield(logged) + logged.save + end + logged end end app/models/logged_mail.rb @@ -5,6 +5,7 @@ class Mapping attr_accessor :default_domain attr_accessor :transports end + self.transports = {"HTTP Post" => 'http_post', "Jabber" => 'jabber'} self.default_domain = 'astrotrain.com' @@ -12,44 +13,82 @@ class Mapping property :user_id, Integer, :nullable => false, :index => true property :email_user, String, :size => 255, :length => 1..255, :index => :email, :format => /^[\w\.\_\%\+\-]*\*?$/ property :email_domain, String, :size => 255, :lenght => 1..255, :index => :email, :format => /^[\w\-\_\.]+$/, :default => lambda { default_domain } - property :destination, String, :size => 255, :length => 1..255, :unique_index => true, :unique => true + property :destination, String, :size => 255, :length => 1..255 property :transport, String, :size => 255, :set => transports.values, :default => 'http_post' property :separator, String, :size => 255 + property :recipient_header_order, String, :size => 255, :auto_validation => false validates_is_unique :email_user, :scope => :email_domain - validates_format :destination, :as => :url, :if => :destination_uses_url? + validates_format :destination, :as => /^(https?:)\/\/[^\/]+\/?/i, :if => :destination_uses_url? validates_format :destination, :as => :email_address, :if => :destination_uses_email? + validates_with_block :recipient_header_order do + if order = recipient_header_order + if !order.all? { |key| Message.recipient_header_order.include?(key) } + [false, "Field should be an array with these choices: delivered_to, original_to, and to"] + else + true + end + else + true + end + end belongs_to :user has n, :logged_mails, :order => [:created_at.desc] - def self.match(email_address) - email_address.strip! - email_address.downcase! - name, domain = email_address.split("@") - match_by_address(name, domain) || match_by_wildcard(name, domain) + def self.match(email_addresses) + email_addresses.each do |email_address| + email_address.strip! + email_address.downcase! + name, domain = email_address.split("@") + if mapping = match_by_address(name, domain) || match_by_wildcard(name, domain) + return [mapping, email_address] + end + end + nil end def self.process(message) - if mapping = match(message.recipient) - mapping.process(message) + LoggedMail.from(message) do |logged| + begin + mapping, recipient = match(message.recipients) + if mapping + logged.recipient = recipient + logged.mapping = mapping + mapping.process(message, recipient) + logged.delivered_at = Time.now.utc + else + FileUtils.rm_rf logged.raw_path + false + end + rescue + logged.error_message = "#{$!.class}: #{$!}" + end end end - def process(message) - Transport.process(message, self) - log_message(message) + def process(message, recipient) + Transport.process(message, self, recipient) end - def log_message(message) - logged = LoggedMail.from(message) - logged_mails << logged - logged.save - logged + def recipient_header_order + if s = attribute_get(:recipient_header_order) + s.split(",") + end + end + + def recipient_header_order=(value) + value = \ + case value + when Array then value * ',' + when String then value + else nil + end + attribute_set(:recipient_header_order, value) end - def match?(name) - name =~ email_user_regex + def match?(name, domain) + email_domain == domain && name =~ email_user_regex end def destination_uses_url? @@ -80,7 +119,9 @@ class Mapping end if last_line - body = lines[0..delim_line] * "\n" + lines = lines[0..delim_line] + strip_date_reply_line_from lines + body = lines * "\n" elsif !delim_line.nil? body = '' end @@ -101,7 +142,17 @@ protected def self.match_by_wildcard(name, domain) wildcards = all(:email_domain => domain, :email_user.like => "%*") wildcards.sort! { |x, y| y.email_user.size <=> x.email_user.size } - wildcards.detect { |w| w.match?(name) } + wildcards.detect { |w| w.match?(name, domain) } + end + + @@language_regexes = [/^on\b.*wrote\b?:$/i, /^am\b.*schrieb [\w\d\s]+:$/i, /^le\b.*a écrit\b?:$/i] + def strip_date_reply_line_from(lines) + @@language_regexes.detect do |lang_re| + if lines.last =~ lang_re + lines.pop + end + end + lines end def email_user_regex app/models/mapping.rb @@ -1,22 +1,16 @@ class Mapping class HttpPost < Transport - @@headers = {'Content-Type' => 'application/json'} - def process return unless Transport.processing - curl_params = post_fields.inject([]) do |params, (key, value)| - value = value * "," if value.is_a?(Array) - params << Curl::PostField.content(key.to_s, value) - end - request.http_post *curl_params + RestClient.post @mapping.destination, fields.merge(:emails => fields[:emails].join(",")) end - def request - @request ||= Curl::Easy.new(@mapping.destination) - end - - def post_fields - @post_fields ||= {:subject => @message.subject, :to => @message.recipient, :from => @message.sender, :body => @message.body} + def fields + super + @message.attachments.each_with_index do |att, index| + @fields[:"attachments_#{index}"] = att + end + @fields end end end \ No newline at end of file app/models/mapping/http_post.rb @@ -14,7 +14,7 @@ class Mapping end def content - @content ||= "From: %s\nTo: %s\nSubject: %s\n%s" % [@message.sender, @message.recipient, @message.subject, @message.body] + @content ||= "From: %s\nTo: %s\nSubject: %s\nEmails: %s\n%s" % [fields[:from], fields[:to], fields[:subject], fields[:emails] * ", ", fields[:body]] end end end \ No newline at end of file app/models/mapping/jabber.rb @@ -7,27 +7,35 @@ class Mapping attr_reader :message, :mapping - def self.process(message, mapping) + def self.process(message, mapping, recipient) case mapping.transport - when 'http_post' then HttpPost.process(message, mapping) - when 'jabber' then Jabber.process(message, mapping) + when 'http_post' then HttpPost.process(message, mapping, recipient) + when 'jabber' then Jabber.process(message, mapping, recipient) end end - def initialize(message, mapping) + def initialize(message, mapping, recipient) message.body = mapping.find_reply_from(message.body) - @message = message - @mapping = mapping + @message = message + @mapping = mapping + @recipient = recipient end def process end + def fields + @fields ||= begin + all_emails = @message.recipients - [@recipient] + {:subject => @message.subject, :to => @recipient, :from => @message.sender, :body => @message.body, :emails => all_emails} + end + end + def self.inherited(child) super class << child - def process(message, mapping) - new(message, mapping).process + def process(message, mapping, recipient) + new(message, mapping, recipient).process end end end app/models/mapping/transport.rb @@ -1,11 +1,15 @@ require 'digest/sha1' require 'fileutils' +require 'tempfile' + +# Wrapper around a TMail object class Message attr_accessor :body - attr_reader :mail + attr_reader :mail, :attachments class << self - attr_reader :queue_path + attr_reader :queue_path, :archive_path + attr_accessor :recipient_header_order end def self.queue_path=(path) @@ -13,6 +17,12 @@ class Message @queue_path = File.expand_path(path) end + def self.archive_path=(path) + FileUtils.mkdir_p path + @archive_path = File.expand_path(path) + end + + self.recipient_header_order = %w(original_to delivered_to to) self.queue_path = File.join(File.dirname(__FILE__), '..', '..', 'queue') def self.queue(raw) @@ -30,6 +40,17 @@ class Message def self.receive(raw) message = parse(raw) Mapping.process(message) + message + end + + def self.receive_file(path, raw = nil) + message = receive IO.read(path) + if archive_path + FileUtils.mv path, archive_path / Time.now.year.to_s / Time.now.month.to_s / Time.now.day.to_s / File.basename(path) + else + FileUtils.rm_rf path + end + message end def self.parse(raw) @@ -37,24 +58,55 @@ class Message end def initialize(mail) - @mail = mail - @mapping = nil + @mail = mail + @mapping = nil + @attachments = [] + @recipients = {} + end + + def recipients(order = nil) + if !@recipients.key?(order) + order = self.class.recipient_header_order if order.blank? + recipients = [] + + parse_email_headers recipients_from_body, recipients + order.each do |key| + parse_email_headers send("recipients_from_#{key}"), recipients + end + + recipients.flatten! + recipients.uniq! + @recipients[order] = recipients + else + @recipients[order] + end + end + + def recipients_from_to + @recipient_from_to ||= [@mail['to'].to_s] end - def recipient - @recipient ||= begin - if value = @mail['X-Original-To'] - value.to_s - elsif value = @mail['Delivered-To'] - value.to_s + def recipients_from_delivered_to + @recipient_from_delivered_to ||= begin + delivered = @mail['Delivered-To'] + if delivered.respond_to?(:first) + delivered.map! { |a| a.to_s } else - @mail['to'].to_s + [delivered.to_s] end end end + def recipients_from_original_to + @recipient_from_original_to ||= [@mail['X-Original-To'].to_s] + end + + def recipients_from_body + @recipients_from_body ||= body.scan(/<[\w\.\_\%\+\-]+@[\w\-\_\.]+>/) + end + def sender - @sender ||= @mail['from'].to_s + @sender ||= TMail::Unquoter.unquote_and_convert_to(@mail['from'].to_s, "utf-8") end def subject @@ -62,10 +114,101 @@ class Message end def body - @body ||= @mail.body + @body ||= begin + if @mail.multipart? + @attachments.clear + @body = [] + scan_parts(@mail) + @body = @body.join("\n") + else + @mail.body + end + end end def raw @mail.port.to_s end + + class Attachment + def initialize(part) + @part = part + @is_read = false + end + + def content_type + @part.content_type + end + + def filename + @filename ||= @part.type_param("name") + end + + # For IO API compatibility when used with Rest-Client + def close + end + + alias path filename + + def read(value = nil) + if read? + nil + else + @is_read = true + data + end + end + + def read? + @is_read == true + end + + def data + @part.body + end + + def attached? + !filename.nil? + end + + def inspect + %(#<Message::Attachment filename=#{filename.inspect} content_type=#{content_type.inspect}>) + end + end + +protected + def scan_parts(message) + message.parts.each do |part| + if part.multipart? + scan_parts(part) + else + if part.content_type == "text/plain" + @body << part.body + else + att = Attachment.new(part) + @attachments << att if att.attached? + end + end + end + end + + def parse_email_headers(values, collection) + values.each do |value| + if !value.blank? + emails = TMail::AddressHeader.new('to', value) + emails.addrs.each do |addr| + email = TMail::Address.parse(addr.to_s) + collection << unescape(email.address) + end + end + end + end + + # Stolen from Rack/Camping, remove the "+" => " " translation + def unescape(s) + s.gsub!(/((?:%[0-9a-fA-F]{2})+)/n){ + [$1.delete('%')].pack('H*') + } + s + end end \ No newline at end of file app/models/message.rb @@ -5,7 +5,7 @@ # You'll need to setup your db as per the salted_user mixin, and you'll need # To use :password, and :password_confirmation when creating a user # -# see config/merb-auth/setup.rb to see how to disable the salted_user mixin +# see merb/merb-auth/setup.rb to see how to disable the salted_user mixin # # You will need to setup your database and create a user. class User app/models/user.rb @@ -1 +1,2 @@ -Autotest.add_discovery { "merb" } \ No newline at end of file +Autotest.add_discovery { "merb" } +Autotest.add_discovery { "rspec" } \ No newline at end of file autotest/discover.rb @@ -15,7 +15,10 @@ class Autotest::Merb < Autotest initialize_test_layout # Ignore any happenings in these directories - add_exception %r%^\./(?:doc|log|public|tmp)% + add_exception %r%^\./(?:doc|log|public|tmp|\.git|\.hg|\.svn|framework|gems|schema|\.DS_Store|autotest|bin|.*\.sqlite3)% + # Ignore SCM directories and custom Autotest mappings + %w[.svn .hg .git .autotest].each { |exception| add_exception(exception) } + # Ignore any mappings that Autotest may have already set up clear_mappings @@ -36,7 +39,7 @@ class Autotest::Merb < Autotest ] end - # Any change to a test or test will cause it to be run + # Any change to a test will cause it to be run add_mapping %r%^test/(unit|models|integration|controllers|views|functional)/.*rb$% do |filename, _| filename end autotest/merb.rb @@ -14,7 +14,7 @@ class Autotest::MerbRspec < Autotest super # Ignore any happenings in these directories - add_exception %r%^\./(?:doc|log|public|tmp|\.git|\.hg|\.svn|framework|gems|schema|\.DS_Store|autotest|bin|.*\.sqlite3)% + add_exception %r%^\./(?:doc|log|public|tmp|\.git|\.hg|\.svn|framework|gems|schema|\.DS_Store|autotest|bin|.*\.sqlite3|.*\.thor)% # Ignore SCM directories and custom Autotest mappings %w[.svn .hg .git .autotest].each { |exception| add_exception(exception) } @@ -44,7 +44,7 @@ class Autotest::MerbRspec < Autotest # Any change to global_helpers will result in all view and controller # tests being run add_mapping %r%^app/helpers/global_helpers\.rb% do - files_matching %r%^spec/(views|controllers|helpers)/.*_spec\.rb$% + files_matching %r%^spec/(views|controllers|helpers|requests)/.*_spec\.rb$% end # Any change to a helper will cause its spec to be run autotest/merb_rspec.rb @@ -1,19 +1,23 @@ -dependency "merb-action-args", "0.9.10" # Provides support for querystring arguments to be passed in to controller actions -dependency "merb-assets", "0.9.10" # Provides link_to, asset_path, auto_link, image_tag methods (and lots more) -dependency "merb-cache", "0.9.10" # Provides your application with caching functions -dependency "merb-helpers", "0.9.10" # Provides the form, date/time, and other helpers -dependency "merb-mailer", "0.9.10" # Integrates mail support via Merb Mailer -dependency "merb-slices", "0.9.10" # Provides a mechanism for letting plugins provide controllers, views, etc. to your app -dependency "merb-auth", "0.9.10" # An authentication slice (Merb's equivalent to Rails' restful authentication) -dependency "merb-param-protection", "0.9.10" +ver = '1.0.7.1' +dependency "merb-action-args", ver # Provides support for querystring arguments to be passed in to controller actions +dependency "merb-assets", ver # Provides link_to, asset_path, auto_link, image_tag methods (and lots more) +dependency "merb-cache", ver # Provides your application with caching functions +dependency "merb-helpers", ver # Provides the form, date/time, and other helpers +dependency "merb-mailer", ver # Integrates mail support via Merb Mailer +dependency "merb-slices", ver # Provides a mechanism for letting plugins provide controllers, views, etc. to your app +dependency "merb-auth", ver # An authentication slice (Merb's equivalent to Rails' restful authentication) +dependency "merb-param-protection", ver -dependency "dm-core", "0.9.6" # The datamapper ORM -dependency "dm-aggregates", "0.9.6" # Provides your DM models with count, sum, avg, min, max, etc. -dependency "dm-migrations", "0.9.6" # Make incremental changes to your database. -dependency "dm-timestamps", "0.9.6" # Automatically populate created_at, created_on, etc. when those properties are present. -dependency "dm-types", "0.9.6" # Provides additional types, including csv, json, yaml. -dependency "dm-validations", "0.9.6" # Validation framework +dm_ver = "0.9.10" +dependency "dm-core", dm_ver # The datamapper ORM +dependency "dm-aggregates", dm_ver # Provides your DM models with count, sum, avg, min, max, etc. +dependency "dm-migrations", dm_ver # Make incremental changes to your database. +dependency "dm-timestamps", dm_ver # Automatically populate created_at, created_on, etc. when those properties are present. +dependency "dm-types", dm_ver # Provides additional types, including csv, json, yaml. +dependency "dm-validations", dm_ver # Validation framework dependency "tmail", "1.2.3.1" -dependency "curb", "0.1.4" dependency "xmpp4r-simple", "0.8.8" + +$LOAD_PATH.unshift File.join(Merb.root, 'vendor', 'rest-client', 'lib') +require 'rest_client' config/dependencies.rb @@ -6,7 +6,7 @@ Merb::Config.use { |c| # log less in testing environment c[:log_level] = :error - c[:log_file] = Merb.root / "log" / "test.log" + #c[:log_file] = Merb.root / "log" / "test.log" # or redirect logger using IO handle - # c[:log_stream] = STDOUT + c[:log_stream] = STDOUT } config/environments/test.rb @@ -1,6 +1,6 @@ # Go to http://wiki.merbivore.com/pages/init-rb -require File.join(File.dirname(__FILE__), 'dependencies.rb') +require 'config/dependencies.rb' use_orm :datamapper use_test :rspec @@ -11,11 +11,13 @@ Merb::Config.use do |c| c[:session_store] = 'cookie' # can also be 'memory', 'memcache', 'container', 'datamapper # cookie session store configuration - c[:session_secret_key] = '1205346b9baa87cf8e49f78124c8d17a31ac0971' # required for cookie session store - # c[:session_id_key] = '_session_id' # cookie session id key, defaults to "_session_id" + c[:session_secret_key] = '6347fdc1ecdb73cb2dce47980ae0b094779575aa' # required for cookie session store + c[:session_id_key] = '_astrotrain_session_id' # cookie session id key, defaults to "_session_id" end Merb::BootLoader.before_app_loads do + TMail::Mail::ALLOW_MULTIPLE['delivered-to'] = true + # This will get executed after dependencies have been loaded but before your app's classes have loaded. end Merb::BootLoader.after_app_loads do config/init.rb @@ -1,4 +1,3 @@ - # use PathPrefix Middleware if :path_prefix is set in Merb::Config if prefix = ::Merb::Config[:path_prefix] use Merb::Rack::PathPrefix, prefix config/rack.rb @@ -15,8 +15,7 @@ namespace :app do realtime = Benchmark.realtime do files = Dir[Merb.root / "queue" / "*"] files.each do |mail| - Message.receive(IO.read(mail)) - File.unlink mail + Message.receive_file(mail) end count = files.size end lib/tasks/astrotrain.rake @@ -14,4 +14,4 @@ RewriteCond %{REQUEST_FILENAME} !-f RewriteRule ^(.*)$ merb.fcgi [QSA,L] -ErrorDocument 500 "<h2>Application Error</h2>Merb could not be reached +ErrorDocument 500 "<h2>Application Error</h2>Merb could not be reached" public/.htaccess @@ -1,5 +1,5 @@ X-Original-To: processor-reply-57@custom.com -Delivered-To: processor@astrotrain.com +Delivered-To: processor-delivered@astrotrain.com Message-ID: <a16be7390810161014n52b603e9k1aa6bb803c6735aa@mail.gmail.com> Date: Thu, 16 Oct 2008 10:14:18 -0700 From: user@example.com, boss@example.com spec/fixtures/custom.txt @@ -3,52 +3,28 @@ require File.join( File.dirname(__FILE__), '..', "spec_helper" ) describe LoggedMail do describe "being created from Message" do before :all do - @raw = mail(:basic) + User.all.destroy! + Mapping.all.destroy! + @user = User.create!(:login => 'user') + @mapping = @user.mappings.create!(:user_id => @user.id, :email_user => '*', :recipient_header_order => 'delivered_to,original_to,to') + @raw = mail(:custom) @message = Message.parse(@raw) - @logged = LoggedMail.from(@message) + @logged = LoggedMail.from(@message) do |l| + l.recipient = @message.recipients(%w(delivered_to)).first + l.mapping = @mapping + end end it "sets recipient" do - @logged.recipient.should == @message.recipient + @logged.recipient.should == @message.recipients(%w(delivered_to)).first end it "sets subject" do @logged.subject.should == @message.subject end - it "sets raw headers" do - @logged.raw.should == @raw - end - end - - describe "logging mapping message" do - before :all do - @raw = mail(:basic) - @message = Message.parse(@raw) - User.transaction do - User.all.destroy! - Mapping.all.destroy! - @user = User.create!(:login => 'user') - @mapping = @user.mappings.create!(:email_user => 'xyz') - @logged = @mapping.log_message @message - end - @logged.reload - end - it "sets mapping" do @logged.mapping.should == @mapping end - - it "sets recipient" do - @logged.recipient.should == @message.recipient - end - - it "sets subject" do - @logged.subject.should == @message.subject - end - - it "sets raw headers" do - @logged.raw.should == @raw - end end end \ No newline at end of file spec/models/logged_mail_spec.rb @@ -17,28 +17,46 @@ describe Mapping do Mapping.new(:transport => 'jabber').destination_uses_email?.should == true end + it "gets comma separated recipient_header_order as array" do + m = Mapping.new + m.attribute_set(:recipient_header_order, "foo,bar") + m.recipient_header_order.should == %w(foo bar) + end + + it "sets comma separated recipient_header_order from array" do + m = Mapping.new + m.recipient_header_order = %w(foo bar) + m.attribute_get(:recipient_header_order).should == "foo,bar" + end + + it "sets comma separated recipient_header_order from string" do + m = Mapping.new + m.recipient_header_order = "foo,bar" + m.attribute_get(:recipient_header_order).should == "foo,bar" + end + describe "matching" do before :all do User.transaction do User.all.destroy! Mapping.all.destroy! @user = User.create!(:login => 'user') - @mapping1 = @user.mappings.create!(:email_user => '*') - @mapping2 = @user.mappings.create!(:email_user => 'abc*') - @mapping3 = @user.mappings.create!(:email_user => 'abc') + @mapping1 = @user.mappings.create!(:user_id => @user.id, :email_user => '*') + @mapping2 = @user.mappings.create!(:user_id => @user.id, :email_user => 'abc*') + @mapping3 = @user.mappings.create!(:user_id => @user.id, :email_user => 'abc') end end it "matches email user" do - Mapping.match("abc@#{Mapping.default_domain}").should == @mapping3 + Mapping.match(["abc@#{Mapping.default_domain}"]).should == [@mapping3, "abc@#{Mapping.default_domain}"] end it "matches email partial wildcard" do - Mapping.match("abc1@#{Mapping.default_domain}").should == @mapping2 + Mapping.match(["abc1@#{Mapping.default_domain}"]).should == [@mapping2, "abc1@#{Mapping.default_domain}"] end it "matches email partial wildcard" do - Mapping.match("def@#{Mapping.default_domain}").should == @mapping1 + Mapping.match(["def@#{Mapping.default_domain}"]).should == [@mapping1, "def@#{Mapping.default_domain}"] end end @@ -51,6 +69,10 @@ describe Mapping do { "foo bar\n\n#{delim}\nfoo" => "foo bar", + "foo bar\n\nOn 13-Jan-09, at 9:17 AM, ENTP Support wrote:\n\n\n#{delim}\nfoo" => "foo bar", + "foo bar\n\nOn Jan 13, 2009 at 2:20 PM, ENTP Support wrote:\n\n\n#{delim}\nfoo" => "foo bar", + "foo bar\n\nAm Tuesday 02 December 2008 schrieb Ricky Bobby:\n\n\n#{delim}\nfoo" => "foo bar", + "foo bar\n\nLe 18 sept. 08 à 09:08, theRemix a écrit :\n\n\n#{delim}\nfoo" => "foo bar", "foo\n bar\nbaz\n\n\n> #{delim}" => "foo\n bar\nbaz", "foo\n\nbar\nbaz\n#{delim}" => "foo", "foo\n bar\nbaz2\n\n\n\n" => "foo\n bar\nbaz2", @@ -68,10 +90,22 @@ describe Mapping do User.all.destroy! Mapping.all.destroy! @user = User.create!(:login => 'user') - @mapping = @user.mappings.create!(:email_user => 'xyz') + @mapping = @user.mappings.create!(:user_id => @user.id, :email_user => 'xyz') end end + it "requires comma separated list" do + valid_mapping(:recipient_header_order => 'a, b').should_not be_valid + end + + it "requires valid choices" do + valid_mapping(:recipient_header_order => 'delivered_to,whatever').should_not be_valid + end + + it "accepts valid header order" do + valid_mapping(:recipient_header_order => 'delivered_to,to,original_to').should be_valid + end + %w(abc abc_def abc-123 abc+def abc%def foo* *).each do |valid| it "allows #email_user == #{valid.inspect}" do valid_mapping(:email_user => valid).should be_valid @@ -96,7 +130,7 @@ describe Mapping do end end - %w(http://example.com https://example.com http://example.com/ http://example.com/foo http://example.com/foo/bar.html http://example.com/foo?blah[baz]=1).each do |valid| + %w(http://example.com https://example.com http://example.com/ http://example.com/foo http://rick:monkey@example.com http://example.com:4567/foo http://localhost:3000/foo http://localhost/foo http://example.com/foo/bar.html http://example.com/foo?blah[baz]=1).each do |valid| it "allows #destination == #{valid.inspect} for http_post transport" do valid_mapping(:destination => valid, :transport => 'http_post').should be_valid end spec/models/mapping_spec.rb @@ -5,20 +5,78 @@ describe Message do describe "against default domain" do before :all do User.transaction do + LoggedMail.all.destroy! User.all.destroy! Mapping.all.destroy! @user = User.create!(:login => 'user') - @mapping = @user.mappings.create!(:email_user => 'xyz') - @mapping2 = @user.mappings.create!(:email_user => 'xyz', :email_domain => 'sample.com') + @mapping = @user.mappings.create!(:user_id => @user.id, :email_user => 'xyz') + @mapping2 = @user.mappings.create!(:user_id => @user.id, :email_user => 'xyz', :email_domain => 'sample.com') end end - it "doesn't log message without mapping" do - lambda { Message.receive(mail(:basic)) }.should_not change(LoggedMail, :count) + describe "without mapping" do + before :all do + @msg = Message.receive(mail(:basic)) + @log = LoggedMail.first + end + + it "doesn't log message" do + @log.should == nil + end end - it "logs message with mapping" do - lambda { Message.receive(mail(:mapped)) }.should change(LoggedMail, :count).by(1) + describe "erroring" do + before do + LoggedMail.all.destroy! + end + + it "logs message without mappping" do + Mapping.stub!(:match).and_raise RuntimeError + @msg = Message.receive(mail(:basic)) + @log = LoggedMail.first + @log.should_not == nil + @log.delivered_at.should == nil + @log.error_message.should =~ /RuntimeError/ + @log.mapping.should == nil + end + + it "logs message without mappping" do + Mapping.stub!(:match).and_return @mapping + @mapping.stub!(:process).and_raise RuntimeError + @msg = Message.receive(mail(:basic)) + @log = LoggedMail.first + @log.should_not == nil + @log.delivered_at.should == nil + @log.error_message.should =~ /RuntimeError/ + @log.mapping.should == @mapping + end + end + + describe "with mapping" do + before :all do + @msg = Message.receive(mail(:mapped)) + @log = LoggedMail.first + end + + it "logs message" do + @log.should_not == nil + end + + it "links mapping" do + @log.mapping.should == @mapping + end + + it "sets subject" do + @log.subject.should == @msg.subject + end + + it "sets recipient" do + @log.recipient.should == @msg.recipients(@mapping.recipient_header_order).first + end + + it "sets delivered_at" do + @log.delivered_at.should_not == nil + end end end end @@ -38,12 +96,11 @@ describe Message do @message.mail.should be_kind_of(TMail::Mail) end - it "recognizes To: header as recipient" do - @message.recipient.should == %("Processor" <processor@astrotrain.com>) + it "recognizes Delivered-To and To: headers as recipients" do + @message.recipients.should == %w(processor@astrotrain.com) end it "recognizes From: header as sender" do - @message.sender.should == %(Bob <user@example.com>) end @@ -59,6 +116,90 @@ describe Message do @message.raw.should == @raw end end + + describe "iso-8859-1 encoded headers" do + before :all do + @raw = mail("iso-8859-1") + @message = Message.parse(@raw) + end + + it "recognizes From: header with strange encoding" do + @message.sender.should == %(Matthéw <user@example.com>) + end + end + + describe "utf-8 encoded headers" do + before :all do + @raw = mail("utf-8") + @message = Message.parse(@raw) + end + + it "recognizes From: header with strange encoding" do + @message.sender.should == %(isnard naiké <user@example.com>) + end + end + + describe "multipart message" do + before :all do + @raw = mail(:multipart) + @message = Message.parse(@raw) + end + + it "#parse parses TMail::Mail object from raw text" do + @message.mail.should be_kind_of(TMail::Mail) + end + + it "recognizes Delivered-To/To: headers as recipient" do + @message.recipients.should == %w(foo@example.com) + end + + it "recognizes message body" do + @message.body.should == "Testing out rich emails with attachments!\n[state:hold responsible:rick]\n\n" + end + + it "retrieves attachments" do + @message.should have(1).attachments + end + + it "retrieves attachment filename" do + @message.attachments.first.filename.should == 'bandit.jpg' + end + + it "retrieves attachment content_type" do + @message.attachments.first.content_type.should == 'image/jpeg' + end + end + + describe "apple multipart message" do + before :all do + @raw = mail(:apple_multipart) + @message = Message.parse(@raw) + end + + it "#parse parses TMail::Mail object from raw text" do + @message.mail.should be_kind_of(TMail::Mail) + end + + it "recognizes To: header as recipient" do + @message.recipients.should == %w(foo@example.com) + end + + it "recognizes message body" do + @message.body.should == "Let's have a test here:\r\n\r\n\r\n\nYum\r\n\r\n\r\nOn Feb 10, 2009, at 3:37 PM, Tender Support wrote:\r\n\r\n> // Add your reply above here\r\n> ==================================================\r\n> From: Tyler Durden\r\n> Subject: Email attachments and file upload\r\n>\r\n> not at the moment ... let me test\r\n>\r\n> View this Discussion online: http://foobar.com\r\n> .\r\n\r\n\r\n\r\n\r\n--Apple-Mail-7-451386929--" + end + + it "retrieves attachments" do + @message.should have(1).attachments + end + + it "retrieves attachment filename" do + @message.attachments.first.filename.should == 'logo.gif' + end + + it "retrieves attachment content_type" do + @message.attachments.first.content_type.should == 'image/gif' + end + end describe "multiple sender/recipients" do before :all do @@ -70,8 +211,12 @@ describe Message do @message.mail.should be_kind_of(TMail::Mail) end - it "recognizes Delivered-to: header as recipient" do - @message.recipient.should == 'processor@astrotrain.com' + it "recognizes To: headers as recipients" do + @message.recipients.should == %w(processor@astrotrain.com other@example.com) + end + + it "recognizes To: headers as recipients with custom header order" do + @message.recipients(%w(to original_to delivered_to)).should == %w(other@example.com processor@astrotrain.com) end it "recognizes From: header as sender" do @@ -91,6 +236,17 @@ describe Message do end end + describe "recipients in the body" do + before :all do + @raw = mail(:multiple_with_body_recipients) + @message = Message.parse(@raw) + end + + it "recognizes in-body emails and To: headers as recipients" do + @message.recipients.should == %w(processor+foobar@astrotrain.com processor+blah@astrotrain.com processor@astrotrain.com other@example.com) + end + end + describe "with x-original-to header" do before :all do @raw = mail(:custom) @@ -101,8 +257,16 @@ describe Message do @message.mail.should be_kind_of(TMail::Mail) end - it "recognizes Delivered-to: header as recipient" do - @message.recipient.should == 'processor-reply-57@custom.com' + it "recognizes X-Original-to: header as recipient" do + @message.recipients.should == %w(processor-reply-57@custom.com processor-delivered@astrotrain.com processor@astrotrain.com) + end + + it "recognizes Delivered-To: header as recipient with custom header order" do + @message.recipients(%w(delivered_to original_to to)).should == %w(processor-delivered@astrotrain.com processor-reply-57@custom.com processor@astrotrain.com) + end + + it "recognizes To: header as recipient with custom header order" do + @message.recipients(%w(to original_to delivered_to)).should == %w(processor@astrotrain.com processor-reply-57@custom.com processor-delivered@astrotrain.com) end it "recognizes From: header as sender" do @@ -121,6 +285,17 @@ describe Message do @message.raw.should == @raw end end + + describe "with multiple delivered-to headers" do + before :all do + @raw = mail(:multiple_delivered_to) + @message = Message.parse(@raw) + end + + it "recognizes Delivered-to: header as recipient" do + @message.recipients.should == %w(processor-reply-57@custom.com processor-delivered@astrotrain.com processor@astrotrain.com) + end + end end describe "queueing" do spec/models/message_spec.rb @@ -3,41 +3,45 @@ require File.join( File.dirname(__FILE__), '..', "spec_helper" ) describe Mapping::HttpPost do before :all do @post = 'http://example.com' - @message = Message.parse(mail(:basic)) + @message = Message.parse(mail(:custom)) @mapping = Mapping.new(:destination => @post, :transport => 'http_post') - @trans = Mapping::HttpPost.new(@message, @mapping) end - it "sets #post_fields" do - @trans.post_fields.should == {:subject => @message.subject, :from => @message.sender, :to => @message.recipient, :body => @message.body} + before do + @trans = Mapping::HttpPost.new(@message, @mapping, @message.recipients(%w(delivered_to)).first) + @expected_fields = @trans.fields.merge(:emails => @message.recipients(%w(original_to to)) * ",") end - it "sets post_fields with mapping separator set" do - @message = Message.parse(mail(:reply)) - @mapping.separator = "=" * 5 - @trans = Mapping::HttpPost.new(@message, @mapping) - @trans.post_fields[:body].should == "blah blah" + it "sets #fields" do + @trans.fields.should == {:subject => @message.subject, :from => @message.sender, :to => @message.recipients(%w(delivered_to)).first, :body => @message.body, :emails => @message.recipients(%w(original_to to))} + end + + it "adds attachments to #fields" do + @multipart = Message.parse(mail(:multipart)) + @trans = Mapping::HttpPost.new(@multipart, @mapping, @multipart.recipients.first) + @trans.fields.should == {:subject => @multipart.subject, :from => @multipart.sender, :to => @multipart.recipients.first, :body => @multipart.body, :attachments_0 => @multipart.attachments.first, :emails => []} end - it "creates request object" do - @trans.request.should be_kind_of(Curl::Easy) - @trans.request.url.should == @post + it "sets fields with mapping separator set" do + @message = Message.parse(mail(:reply)) + @mapping.separator = "=" * 5 + @trans = Mapping::HttpPost.new(@message, @mapping, @message.recipients.first) + @trans.fields[:body].should == "blah blah" end describe "when processing" do before do - @trans.request.stub!(:http_post) Mapping::HttpPost.stub!(:new).and_return(@trans) end it "makes http post request" do - @trans.request.should_receive :http_post + RestClient.should_receive(:post).with(@mapping.destination, @expected_fields) @trans.process end it "makes http post request from Transport" do - @trans.request.should_receive :http_post - Mapping::Transport.process(@message, @mapping) + RestClient.should_receive(:post).with(@mapping.destination, @expected_fields) + Mapping::Transport.process(@message, @mapping, @message.recipients.first) end before :all do @@ -53,20 +57,23 @@ end describe Mapping::Jabber do before :all do @dest = 'foo@bar.com' - @message = Message.parse(mail(:basic)) + @message = Message.parse(mail(:custom)) @mapping = Mapping.new(:destination => @dest, :transport => 'jabber') - @trans = Mapping::Jabber.new(@message, @mapping) + end + + before do + @trans = Mapping::Jabber.new(@message, @mapping, @message.recipients(%w(delivered_to)).first) end it "sets #content" do - @trans.content.should == "From: %s\nTo: %s\nSubject: %s\n%s" % [@message.sender, @message.recipient, @message.subject, @message.body] + @trans.content.should == "From: %s\nTo: %s\nSubject: %s\nEmails: %s\n%s" % [@message.sender, @message.recipients(%w(delivered_to)).first, @message.subject, @message.recipients(%w(original_to to)) * ", ", @message.body] end it "sets content with mapping separator set" do @message = Message.parse(mail(:reply)) @mapping.separator = "=" * 5 - @trans = Mapping::Jabber.new(@message, @mapping) - @trans.content.should == "From: %s\nTo: %s\nSubject: %s\n%s" % [@message.sender, @message.recipient, @message.subject, "blah blah"] + @trans = Mapping::Jabber.new(@message, @mapping, @message.recipients(%w(delivered_to)).first) + @trans.content.should == "From: %s\nTo: %s\nSubject: %s\nEmails: %s\n%s" % [@message.sender, @message.recipients(%w(delivered_to)).first, @message.subject, '', "blah blah"] end describe "when processing" do @@ -83,7 +90,7 @@ describe Mapping::Jabber do it "makes jabber delivery from Transport" do @trans.connection.should_receive :deliver - Mapping::Transport.process(@message, @mapping) + Mapping::Transport.process(@message, @mapping, @message.recipients(%w(delivered_to)).first) end before :all do spec/models/transport_spec.rb @@ -23,6 +23,10 @@ Spec::Runner.configure do |config| end end +LoggedMail.log_path = Merb.root / 'spec' / 'messages' +FileUtils.rm_rf LoggedMail.log_path +FileUtils.mkdir_p LoggedMail.log_path + begin require 'ruby-debug' Debugger.start spec/spec_helper.rb @@ -5,7 +5,7 @@ require 'fileutils' require 'yaml' # Important - don't change this line or its position -MERB_THOR_VERSION = '0.0.5' +MERB_THOR_VERSION = '0.2.1' ############################################################################## @@ -92,6 +92,7 @@ module GemManagement if installer.installed_gems.empty? && exception error "Failed to install gem '#{gem} (#{version || 'any version'})' (#{exception.message})" end + ensure_bin_wrapper_for_installed_gems(installer.installed_gems, options) installer.installed_gems.each do |spec| success "Successfully installed #{spec.full_name}" end @@ -119,6 +120,7 @@ module GemManagement if installer.installed_gems.empty? && exception error "Failed to install gem '#{gem}' (#{e.message})" end + ensure_bin_wrapper_for_installed_gems(installer.installed_gems, options) installer.installed_gems.each do |spec| success "Successfully installed #{spec.full_name}" end @@ -132,8 +134,8 @@ module GemManagement # install_gem_from_source(source_dir, :skip => [...]) def install_gem_from_source(source_dir, *args) installed_gems = [] - Dir.chdir(source_dir) do - opts = args.last.is_a?(Hash) ? args.pop : {} + opts = args.last.is_a?(Hash) ? args.pop : {} + Dir.chdir(source_dir) do gem_name = args[0] || File.basename(source_dir) gem_pkg_dir = File.join(source_dir, 'pkg') gem_pkg_glob = File.join(gem_pkg_dir, "#{gem_name}-*.gem") @@ -173,6 +175,8 @@ module GemManagement end end + ensure_bin_wrapper_for(opts[:install_dir], opts[:bin_dir], *installed_gems) + # Finally install the main gem if install_pkg(Dir[gem_pkg_glob][0], opts.merge(:refresh => refresh)) installed_gems = refresh @@ -198,7 +202,7 @@ module GemManagement options[:version] = Gem::Requirement.new ["= #{options[:version]}"] end update_source_index(options[:install_dir]) if options[:install_dir] - Gem::Uninstaller.new(gem, options).uninstall + Gem::Uninstaller.new(gem, options).uninstall rescue nil end def clobber(source_dir) @@ -260,7 +264,7 @@ module GemManagement gemspecs = ::Gem.source_index.search(dep) local = gemspecs.reverse.find { |s| s.loaded_from.index(gem_dir) == 0 } if local - local_specs << local + local_specs << local elsif gemspecs.last system_specs << gemspecs.last else @@ -268,6 +272,15 @@ module GemManagement end end ::Gem.clear_paths + else + dependencies.each do |dep| + gemspecs = ::Gem.source_index.search(dep) + if gemspecs.last + system_specs << gemspecs.last + else + missing_deps << dep + end + end end [system_specs, local_specs, missing_deps] end @@ -292,7 +305,14 @@ module GemManagement end end end - + + def ensure_bin_wrapper_for_installed_gems(gemspecs, options) + if options[:install_dir] && options[:bin_dir] + gems = gemspecs.map { |spec| spec.name } + ensure_bin_wrapper_for(options[:install_dir], options[:bin_dir], *gems) + end + end + private def executable_wrapper(spec, bin_file_name, minigems = true) @@ -313,12 +333,16 @@ rescue LoadError require '#{then_req}' end -if File.directory?(gems_dir = File.join(Dir.pwd, 'gems')) || - File.directory?(gems_dir = File.join(File.dirname(__FILE__), '..', 'gems')) - $BUNDLE = true; Gem.clear_paths; Gem.path.unshift(gems_dir) +# use gems dir if ../gems exists - eg. only for ./bin/#{bin_file_name} +if File.directory?(gems_dir = File.join(File.dirname(__FILE__), '..', 'gems')) + $BUNDLE = true; Gem.clear_paths; Gem.path.replace([File.expand_path(gems_dir)]) + ENV["PATH"] = "\#{File.dirname(__FILE__)}:\#{gems_dir}/bin:\#{ENV["PATH"]}" + if (local_gem = Dir[File.join(gems_dir, "specifications", "#{spec.name}-*.gemspec")].last) + version = File.basename(local_gem)[/-([\\.\\d]+)\\.gemspec$/, 1] + end end -version = "#{Gem::Requirement.default}" +version ||= "#{Gem::Requirement.default}" if ARGV.first =~ /^_(.*)_$/ and Gem::Version.correct? $1 then version = $1 @@ -415,16 +439,15 @@ end module MerbThorHelper - attr_accessor :include_dependencies + attr_accessor :force_gem_dir def self.included(base) base.send(:include, ColorfulMessages) base.extend ColorfulMessages end - - def install_dependency(dependency, opts = {}) - opts[:version] ||= dependency.version_requirements.to_s - Merb::Gem.install(dependency.name, default_install_options.merge(opts)) + + def use_edge_gem_server + ::Gem.sources << 'http://edge.merbivore.com' end def source_manager @@ -467,6 +490,12 @@ module MerbThorHelper end end end + + def install_dependency(dependency, opts = {}) + version = dependency.version_requirements.to_s + install_opts = default_install_options.merge(:version => version) + Merb::Gem.install(dependency.name, install_opts.merge(opts)) + end def install_dependency_from_source(dependency, opts = {}) matches = Dir[File.join(source_dir, "**", dependency.name, "{Rakefile,Thorfile}")] @@ -506,7 +535,7 @@ module MerbThorHelper note 'Uninstalling existing local gems:' local.each { |gemspec| note "Uninstalled #{gemspec.name}" } else - message 'Uninstalling existing local gems:' + message 'Uninstalling existing local gems:' if local.size > 1 local.each do |gemspec| Merb::Gem.uninstall(gemspec.name, default_uninstall_options) end @@ -538,11 +567,11 @@ module MerbThorHelper end def default_install_options - { :install_dir => gem_dir, :ignore_dependencies => ignore_dependencies? } + { :install_dir => gem_dir, :bin_dir => bin_dir, :ignore_dependencies => ignore_dependencies? } end def default_uninstall_options - { :install_dir => gem_dir, :ignore => true, :all => true, :executables => true } + { :install_dir => gem_dir, :bin_dir => bin_dir, :ignore => true, :all => true, :executables => true } end def dry_run? @@ -550,11 +579,7 @@ module MerbThorHelper end def ignore_dependencies? - options[:"ignore-dependencies"] || !include_dependencies? - end - - def include_dependencies? - options[:"include-dependencies"] || self.include_dependencies + options[:"ignore-dependencies"] end # The current working directory, or Merb app root (--merb-root option). @@ -571,6 +596,7 @@ module MerbThorHelper # If a local ./gems dir is found, return it. def gem_dir + return force_gem_dir if force_gem_dir if File.directory?(dir = default_gem_dir) dir end @@ -596,16 +622,12 @@ module MerbThorHelper def create_if_missing(path) FileUtils.mkdir(path) unless File.exists?(path) end - - def ensure_bin_wrapper_for(*gems) - Merb::Gem.ensure_bin_wrapper_for(gem_dir, bin_dir, *gems) - end def sudo ENV['THOR_SUDO'] ||= "sudo" sudo = Gem.win_platform? ? "" : ENV['THOR_SUDO'] end - + def local_gemspecs(directory = gem_dir) if File.directory?(specs_dir = File.join(directory, 'specifications')) Dir[File.join(specs_dir, '*.gemspec')].map do |gemspec_path| @@ -625,23 +647,538 @@ end $SILENT = true # don't output all the mess some rake package tasks spit out module Merb + + class Gem < Thor + + include MerbThorHelper + extend GemManagement + + attr_accessor :system, :local, :missing + global_method_options = { + "--merb-root" => :optional, # the directory to operate on + "--version" => :optional, # gather specific version of gem + "--ignore-dependencies" => :boolean # don't install sub-dependencies + } + + method_options global_method_options + def initialize(*args); super; end + + # List gems that match the specified criteria. + # + # By default all local gems are listed. When the first argument is 'all' the + # list is partitioned into system an local gems; specify 'system' to show + # only system gems. A second argument can be used to filter on a set of known + # components, like all merb-more gems for example. + # + # Examples: + # + # merb:gem:list # list all local gems - the default + # merb:gem:list all # list system and local gems + # merb:gem:list system # list only system gems + # merb:gem:list all merb-more # list only merb-more related gems + # merb:gem:list --version 0.9.8 # list gems that match the version + + desc 'list [all|local|system] [comp]', 'Show installed gems' + def list(filter = 'local', comp = nil) + deps = comp ? Merb::Stack.select_component_dependencies(dependencies, comp) : dependencies + self.system, self.local, self.missing = Merb::Gem.partition_dependencies(deps, gem_dir) + case filter + when 'all' + message 'Installed system gems:' + display_gemspecs(system) + message 'Installed local gems:' + display_gemspecs(local) + when 'system' + message 'Installed system gems:' + display_gemspecs(system) + when 'local' + message 'Installed local gems:' + display_gemspecs(local) + else + warning "Invalid listing filter '#{filter}'" + end + end + + # Install the specified gems. + # + # All arguments should be names of gems to install. + # + # When :force => true then any existing versions of the gems to be installed + # will be uninstalled first. It's important to note that so-called meta-gems + # or gems that exactly match a set of Merb::Stack.components will have their + # sub-gems uninstalled too. For example, uninstalling merb-more will install + # all contained gems: merb-action-args, merb-assets, merb-gen, ... + # + # Examples: + # + # merb:gem:install merb-core merb-slices # install all specified gems + # merb:gem:install merb-core --version 0.9.8 # install a specific version of a gem + # merb:gem:install merb-core --force # uninstall then subsequently install the gem + # merb:gem:install merb-core --cache # try to install locally from system gems + # merb:gem:install merb --merb-edge # install from edge.merbivore.com + + desc 'install GEM_NAME [GEM_NAME, ...]', 'Install a gem from rubygems' + method_options "--cache" => :boolean, + "--dry-run" => :boolean, + "--force" => :boolean, + "--merb-edge" => :boolean + def install(*names) + opts = { :version => options[:version], :cache => options[:cache] } + use_edge_gem_server if options[:"merb-edge"] + current_gem = nil + + # uninstall existing gems of the ones we're going to install + uninstall(*names) if options[:force] + + message "Installing #{names.length} #{names.length == 1 ? 'gem' : 'gems'}..." + puts "This may take a while..." + + names.each do |gem_name| + current_gem = gem_name + if dry_run? + note "Installing #{current_gem}..." + else + message "Installing #{current_gem}..." + self.class.install(gem_name, default_install_options.merge(opts)) + end + end + rescue => e + error "Failed to install #{current_gem ? current_gem : 'gem'} (#{e.message})" + end + + # Uninstall the specified gems. + # + # By default all specified gems are uninstalled. It's important to note that + # so-called meta-gems or gems that match a set of Merb::Stack.components will + # have their sub-gems uninstalled too. For example, uninstalling merb-more + # will install all contained gems: merb-action-args, merb-assets, ... + # + # Existing dependencies will be clobbered; when :force => true then all gems + # will be cleared, otherwise only existing local dependencies of the + # matching component set will be removed. + # + # Examples: + # + # merb:gem:uninstall merb-core merb-slices # uninstall all specified gems + # merb:gem:uninstall merb-core --version 0.9.8 # uninstall a specific version of a gem + + desc 'uninstall GEM_NAME [GEM_NAME, ...]', 'Unstall a gem' + method_options "--dry-run" => :boolean + def uninstall(*names) + opts = { :version => options[:version] } + current_gem = nil + if dry_run? + note "Uninstalling any existing gems of: #{names.join(', ')}" + else + message "Uninstalling any existing gems of: #{names.join(', ')}" + names.each do |gem_name| + current_gem = gem_name + Merb::Gem.uninstall(gem_name, default_uninstall_options) rescue nil + # if this gem is a meta-gem or a component set name, remove sub-gems + (Merb::Stack.components(gem_name) || []).each do |comp| + Merb::Gem.uninstall(comp, default_uninstall_options) rescue nil + end + end + end + rescue => e + error "Failed to uninstall #{current_gem ? current_gem : 'gem'} (#{e.message})" + end + + # Recreate all gems from gems/cache on the current platform. + # + # This task should be executed as part of a deployment setup, where the + # deployment system runs this after the app has been installed. + # Usually triggered by Capistrano, God... + # + # It will regenerate gems from the bundled gems cache for any gem that has + # C extensions - which need to be recompiled for the target deployment platform. + # + # Note: at least gems/cache and gems/specifications should be in your SCM. + + desc 'redeploy', 'Recreate all gems on the current platform' + method_options "--dry-run" => :boolean, "--force" => :boolean + def redeploy + require 'tempfile' # for Dir::tmpdir access + if gem_dir && File.directory?(cache_dir = File.join(gem_dir, 'cache')) + specs = local_gemspecs + message "Recreating #{specs.length} gems from cache..." + puts "This may take a while..." + specs.each do |gemspec| + if File.exists?(gem_file = File.join(cache_dir, "#{gemspec.full_name}.gem")) + gem_file_copy = File.join(Dir::tmpdir, File.basename(gem_file)) + if dry_run? + note "Recreating #{gemspec.full_name}" + else + message "Recreating #{gemspec.full_name}" + if options[:force] && File.directory?(gem = File.join(gem_dir, 'gems', gemspec.full_name)) + puts "Removing existing #{gemspec.full_name}" + FileUtils.rm_rf(gem) + end + # Copy the gem to a temporary file, because otherwise RubyGems/FileUtils + # will complain about copying identical files (same source/destination). + FileUtils.cp(gem_file, gem_file_copy) + Merb::Gem.install(gem_file_copy, :install_dir => gem_dir, :ignore_dependencies => true) + File.delete(gem_file_copy) + end + end + end + else + error "No application local gems directory found" + end + end + + private + + # Return dependencies for all installed gems; both system-wide and locally; + # optionally filters on :version requirement. + def dependencies + version_req = if options[:version] + ::Gem::Requirement.create(options[:version]) + else + ::Gem::Requirement.default + end + if gem_dir + ::Gem.clear_paths; ::Gem.path.unshift(gem_dir) + ::Gem.source_index.refresh! + end + deps = [] + ::Gem.source_index.each do |fullname, gemspec| + if version_req.satisfied_by?(gemspec.version) + deps << ::Gem::Dependency.new(gemspec.name, "= #{gemspec.version}") + end + end + ::Gem.clear_paths if gem_dir + deps.sort + end + + public + + # Install gem with some default options. + def self.install(name, options = {}) + defaults = {} + defaults[:cache] = false unless opts[:install_dir] + install_gem(name, defaults.merge(options)) + end + + # Uninstall gem with some default options. + def self.uninstall(name, options = {}) + defaults = { :ignore => true, :executables => true } + uninstall_gem(name, defaults.merge(options)) + end + + end + + class Tasks < Thor + + include MerbThorHelper + + # Show merb.thor version information + # + # merb:tasks:version # show the current version info + # merb:tasks:version --info # show extended version info + + desc 'version', 'Show verion info' + method_options "--info" => :boolean + def version + message "Currently installed merb.thor version: #{MERB_THOR_VERSION}" + if options[:version] + self.options = { :"dry-run" => true } + self.update # run update task with dry-run enabled + end + end + + # Update merb.thor tasks from remotely available version + # + # merb:tasks:update # update merb.thor + # merb:tasks:update --force # force-update merb.thor + # merb:tasks:update --dry-run # show version info only + + desc 'update [URL]', 'Fetch the latest merb.thor and install it locally' + method_options "--dry-run" => :boolean, "--force" => :boolean + def update(url = 'http://merbivore.com/merb.thor') + require 'open-uri' + require 'rubygems/version' + remote_file = open(url) + code = remote_file.read + + # Extract version information from the source code + if version = code[/^MERB_THOR_VERSION\s?=\s?('|")([\.\d]+)('|")/,2] + # borrow version comparison from rubygems' Version class + current_version = ::Gem::Version.new(MERB_THOR_VERSION) + remote_version = ::Gem::Version.new(version) + + if current_version >= remote_version + puts "currently installed: #{current_version}" + if current_version != remote_version + puts "available version: #{remote_version}" + end + info "No update of merb.thor necessary#{options[:force] ? ' (forced)' : ''}" + proceed = options[:force] + elsif current_version < remote_version + puts "currently installed: #{current_version}" + puts "available version: #{remote_version}" + proceed = true + end + + if proceed && !dry_run? + File.open(File.join(__FILE__), 'w') do |f| + f.write(code) + end + success "Installed the latest merb.thor (v#{version})" + end + else + raise "invalid source-code data" + end + rescue OpenURI::HTTPError + error "Error opening #{url}" + rescue => e + error "An error occurred (#{e.message})" + end + + end + + #### MORE LOW-LEVEL TASKS #### + + class Source < Thor + + group 'core' + + include MerbThorHelper + extend GemManagement + + attr_accessor :system, :local, :missing + + global_method_options = { + "--merb-root" => :optional, # the directory to operate on + "--ignore-dependencies" => :boolean, # don't install sub-dependencies + "--sources" => :optional # a yml config to grab sources from + } + + method_options global_method_options + def initialize(*args); super; end + + # List source repositories, of either local or known sources. + # + # Examples: + # + # merb:source:list # list all local sources + # merb:source:list available # list all known sources + + desc 'list [local|available]', 'Show git source repositories' + def list(mode = 'local') + if mode == 'available' + message 'Available source repositories:' + repos = self.class.repos(options[:sources]) + repos.keys.sort.each { |name| puts "- #{name}: #{repos[name]}" } + elsif mode == 'local' + message 'Current source repositories:' + Dir[File.join(source_dir, '*')].each do |src| + next unless File.directory?(src) + src_name = File.basename(src) + unless (repos = source_manager.existing_repos(src_name)).empty? + puts "#{src_name}" + repos.keys.sort.each { |b| puts "- #{b}: #{repos[b]}" } + end + end + else + error "Unknown listing: #{mode}" + end + end + + # Install the specified gems. + # + # All arguments should be names of gems to install. + # + # When :force => true then any existing versions of the gems to be installed + # will be uninstalled first. It's important to note that so-called meta-gems + # or gems that exactly match a set of Merb::Stack.components will have their + # sub-gems uninstalled too. For example, uninstalling merb-more will install + # all contained gems: merb-action-args, merb-assets, merb-gen, ... + # + # Examples: + # + # merb:source:install merb-core merb-slices # install all specified gems + # merb:source:install merb-core --force # uninstall then subsequently install the gem + # merb:source:install merb-core --wipe # clear repo then install the gem + + desc 'install GEM_NAME [GEM_NAME, ...]', 'Install a gem from git source/edge' + method_options "--dry-run" => :boolean, + "--force" => :boolean, + "--wipe" => :boolean + def install(*names) + use_edge_gem_server + # uninstall existing gems of the ones we're going to install + uninstall(*names) if options[:force] || options[:wipe] + + # We want dependencies instead of just names + deps = names.map { |n| ::Gem::Dependency.new(n, ::Gem::Requirement.default) } + + # Selectively update repositories for the matching dependencies + update_dependency_repositories(deps) unless dry_run? + + current_gem = nil + deps.each do |dependency| + current_gem = dependency.name + if dry_run? + note "Installing #{current_gem} from source..." + else + message "Installing #{current_gem} from source..." + puts "This may take a while..." + unless install_dependency_from_source(dependency) + raise "gem source not found" + end + end + end + rescue => e + error "Failed to install #{current_gem ? current_gem : 'gem'} (#{e.message})" + end + + # Uninstall the specified gems. + # + # By default all specified gems are uninstalled. It's important to note that + # so-called meta-gems or gems that match a set of Merb::Stack.components will + # have their sub-gems uninstalled too. For example, uninstalling merb-more + # will install all contained gems: merb-action-args, merb-assets, ... + # + # Existing dependencies will be clobbered; when :force => true then all gems + # will be cleared, otherwise only existing local dependencies of the + # matching component set will be removed. Additionally when :wipe => true, + # the matching git repositories will be removed from the source directory. + # + # Examples: + # + # merb:source:uninstall merb-core merb-slices # uninstall all specified gems + # merb:source:uninstall merb-core --wipe # force-uninstall a gem and clear repo + + desc 'uninstall GEM_NAME [GEM_NAME, ...]', 'Unstall a gem (specify --force to remove the repo)' + method_options "--version" => :optional, "--dry-run" => :boolean, "--wipe" => :boolean + def uninstall(*names) + # Remove the repos that contain the gem + if options[:wipe] + extract_repositories(names).each do |(name, url)| + if File.directory?(src = File.join(source_dir, name)) + if dry_run? + note "Removing #{src}..." + else + info "Removing #{src}..." + FileUtils.rm_rf(src) + end + end + end + end + + # Use the Merb::Gem#uninstall task to handle this + gem_tasks = Merb::Gem.new + gem_tasks.options = options + gem_tasks.uninstall(*names) + end + + # Update the specified source repositories. + # + # The arguments can be actual repository names (from Merb::Source.repos) + # or names of known merb stack gems. If the repo doesn't exist already, + # it will be created and cloned. + # + # merb:source:pull merb-core # update source of specified gem + # merb:source:pull merb-slices # implicitly updates merb-more + + desc 'pull REPO_NAME [GEM_NAME, ...]', 'Update git source repository from edge' + def pull(*names) + repos = extract_repositories(names) + update_repositories(repos) + unless repos.empty? + message "Updated the following repositories:" + repos.each { |name, url| puts "- #{name}: #{url}" } + else + warning "No repositories found to update!" + end + end + + # Clone a git repository into ./src. + + # The repository can be a direct git url or a known -named- repository. + # + # Examples: + # + # merb:source:clone merb-core + # merb:source:clone dm-core awesome-repo + # merb:source:clone dm-core --sources ./path/to/sources.yml + # merb:source:clone git://github.com/sam/dm-core.git + + desc 'clone (REPO_NAME|URL) [DIR_NAME]', 'Clone git source repository by name or url' + def clone(repository, name = nil) + if repository =~ /^git:\/\// + repository_url = repository + repository_name = File.basename(repository_url, '.git') + elsif url = Merb::Source.repo(repository, options[:sources]) + repository_url = url + repository_name = repository + end + source_manager.clone(name || repository_name, repository_url) + end + + # Git repository sources - pass source_config option to load a yaml + # configuration file - defaults to ./config/git-sources.yml and + # ~/.merb/git-sources.yml - which you need to create yourself. + # + # Example of contents: + # + # merb-core: git://github.com/myfork/merb-core.git + # merb-more: git://github.com/myfork/merb-more.git + + def self.repos(source_config = nil) + source_config ||= begin + local_config = File.join(Dir.pwd, 'config', 'git-sources.yml') + user_config = File.join(ENV["HOME"] || ENV["APPDATA"], '.merb', 'git-sources.yml') + File.exists?(local_config) ? local_config : user_config + end + if source_config && File.exists?(source_config) + default_repos.merge(YAML.load(File.read(source_config))) + else + default_repos + end + end + + def self.repo(name, source_config = nil) + self.repos(source_config)[name] + end + + # Default Git repositories + def self.default_repos + @_default_repos ||= { + 'merb' => "git://github.com/wycats/merb.git", + 'merb-plugins' => "git://github.com/wycats/merb-plugins.git", + 'extlib' => "git://github.com/sam/extlib.git", + 'dm-core' => "git://github.com/sam/dm-core.git", + 'dm-more' => "git://github.com/sam/dm-more.git", + 'sequel' => "git://github.com/wayneeseguin/sequel.git", + 'do' => "git://github.com/sam/do.git", + 'thor' => "git://github.com/wycats/thor.git", + 'rake' => "git://github.com/jimweirich/rake.git" + } + end + + end + class Dependencies < Thor + + group 'core' # The Dependencies tasks will install dependencies based on actual application # dependencies. For this, the application is queried for any dependencies. # All operations will be performed within this context. - attr_accessor :system, :local, :missing + attr_accessor :system, :local, :missing, :extract_dependencies include MerbThorHelper global_method_options = { "--merb-root" => :optional, # the directory to operate on - "--include-dependencies" => :boolean, # gather sub-dependencies + "--ignore-dependencies" => :boolean, # ignore sub-dependencies "--stack" => :boolean, # gather only stack dependencies "--no-stack" => :boolean, # gather only non-stack dependencies - "--config" => :boolean, # gather dependencies from yaml config + "--extract" => :boolean, # gather dependencies from the app itself "--config-file" => :optional, # gather from the specified yaml config "--version" => :optional # gather specific version of framework } @@ -663,7 +1200,7 @@ module Merb # merb:dependencies:list all merb-more # list only merb-more related dependencies # merb:dependencies:list --stack # list framework dependencies # merb:dependencies:list --no-stack # list 3rd party dependencies - # merb:dependencies:list --config # list dependencies from the default config + # merb:dependencies:list --extract # list dependencies by extracting them # merb:dependencies:list --config-file file.yml # list from the specified config file desc 'list [all|local|system|missing] [comp]', 'Show application dependencies' @@ -723,7 +1260,7 @@ module Merb # merb:dependencies:install stable merb-more # install only merb-more related dependencies # merb:dependencies:install stable --stack # install framework dependencies # merb:dependencies:install stable --no-stack # install 3rd party dependencies - # merb:dependencies:install stable --config # read dependencies from the default config + # merb:dependencies:install stable --extract # extract dependencies from the actual app # merb:dependencies:install stable --config-file file.yml # read from the specified config file # # In addition to the options above, edge install uses the following: @@ -747,8 +1284,10 @@ module Merb if only_missing = comp == 'missing' message "Preparing to install missing gems #{where} using #{strategy} strategy..." comp = nil + clobber = false else message "Preparing to install #{where} using #{strategy} strategy..." + clobber = true end # If comp given, filter on known stack components @@ -764,7 +1303,8 @@ module Merb warning "No dependencies to install..." else puts "#{deps.length} dependencies to install..." - install_dependencies(strategy, deps) + puts "This may take a while..." + install_dependencies(strategy, deps, clobber) end # Show current dependency info now that we're done @@ -792,7 +1332,6 @@ module Merb # # merb:dependencies:uninstall # uninstall all dependencies - the default # merb:dependencies:uninstall merb-more # uninstall merb-more related gems locally - # merb:dependencies:uninstall --config # read dependencies from the default config desc 'uninstall [comp]', 'Uninstall application dependencies' method_options "--dry-run" => :boolean, "--force" => :boolean @@ -804,42 +1343,17 @@ module Merb clobber_dependencies! end - # Recreate binary gems on the current platform. - # - # This task should be executed as part of a deployment setup, where the - # deployment system runs this after the app has been installed. - # Usually triggered by Capistrano, God... - # - # It will regenerate gems from the bundled gems cache for any gem that has - # C extensions - which need to be recompiled for the target deployment platform. + # Recreate all gems from gems/cache on the current platform. # - # Note: gems/cache should be in your SCM for this to work correctly. + # Note: use merb:gem:redeploy instead - desc 'redeploy', 'Recreate any binary gems on the target platform' - method_options "--dry-run" => :boolean + desc 'redeploy', 'Recreate all gems on the current platform' + method_options "--dry-run" => :boolean, "--force" => :boolean def redeploy - require 'tempfile' # for Dir::tmpdir access - if gem_dir && File.directory?(cache_dir = File.join(gem_dir, 'cache')) - local_gemspecs.each do |gemspec| - unless gemspec.extensions.empty? - if File.exists?(gem_file = File.join(cache_dir, "#{gemspec.full_name}.gem")) - gem_file_copy = File.join(Dir::tmpdir, File.basename(gem_file)) - if dry_run? - note "Recreating #{gemspec.full_name}" - else - message "Recreating #{gemspec.full_name}" - # Copy the gem to a temporary file, because otherwise RubyGems/FileUtils - # will complain about copying identical files (same source/destination). - FileUtils.cp(gem_file, gem_file_copy) - Merb::Gem.install(gem_file_copy, :install_dir => gem_dir) - File.delete(gem_file_copy) - end - end - end - end - else - error "No application local gems directory found" - end + warning 'Warning: merb:dependencies:redeploy has been deprecated - use merb:gem:redeploy instead' + gem = Merb::Gem.new + gem.options = options + gem.redeploy end # Create a dependencies configuration file. @@ -860,10 +1374,22 @@ module Merb # merb:dependencies:configure --config-file file.yml # write to the specified config file desc 'configure [comp]', 'Create a dependencies config file' - method_options "--dry-run" => :boolean, "--force" => :boolean + method_options "--dry-run" => :boolean, "--force" => :boolean, "--versions" => :boolean def configure(comp = nil) + self.extract_dependencies = true # of course we need to consult the app itself # If comp given, filter on known stack components deps = comp ? Merb::Stack.select_component_dependencies(dependencies, comp) : dependencies + + # If --versions is set, update the version_requirements with the actual version available + if options[:versions] + specs = local_gemspecs + deps.each do |dep| + if spec = specs.find { |s| s.name == dep.name } + dep.version_requirements = ::Gem::Requirement.create(spec.version) + end + end + end + config = YAML.dump(deps.map { |d| d.to_s }) puts "#{config}\n" if File.exists?(config_file) && !options[:force] @@ -889,50 +1415,41 @@ module Merb end end - def install_dependencies(strategy, deps) + def install_dependencies(strategy, deps, clobber = true) if method = strategy?(strategy) # Clobber existing local dependencies - clobber_dependencies! - + clobber_dependencies! if clobber + # Run the chosen strategy - collect files installed from stable gems installed_from_stable = send(method, deps).map { |d| d.name } - - # Sleep a bit otherwise the following steps won't see the new files - sleep(deps.length) if deps.length > 0 - - # Leave a file to denote the strategy that has been used for this dependency - self.local.each do |spec| - next unless File.directory?(spec.full_gem_path) - unless installed_from_stable.include?(spec.name) - FileUtils.touch(File.join(spec.full_gem_path, "#{strategy}.strategy")) - else - FileUtils.touch(File.join(spec.full_gem_path, "stable.strategy")) - end - end - - # Ask for system installation of minigem - needs sudo... - if deps.find { |d| d.name == 'minigems' } - info "Installing minigems.rb on your system..." - `#{sudo} minigem install` - end + + unless dry_run? + # Sleep a bit otherwise the following steps won't see the new files + sleep(deps.length) if deps.length > 0 && deps.length <= 10 - # Add local binaries for the installed framework dependencies - comps = Merb::Stack.all_components & deps.map { |d| d.name } - comps << { :no_minigems => 'merb-gen' } - ensure_bin_wrapper_for(*comps) + # Leave a file to denote the strategy that has been used for this dependency + self.local.each do |spec| + next unless File.directory?(spec.full_gem_path) + unless installed_from_stable.include?(spec.name) + FileUtils.touch(File.join(spec.full_gem_path, "#{strategy}.strategy")) + else + FileUtils.touch(File.join(spec.full_gem_path, "stable.strategy")) + end + end + end return true end false end def dependencies - if use_config? - # Use preconfigured dependencies from yaml file - deps = config_dependencies - else + if extract_dependencies? # Extract dependencies from the current application deps = Merb::Stack.core_dependencies(gem_dir, ignore_dependencies?) - deps += Merb::Dependencies.extract_dependencies(working_dir) + deps += Merb::Dependencies.extract_dependencies(working_dir) + else + # Use preconfigured dependencies from yaml file + deps = config_dependencies end stack_components = Merb::Stack.components @@ -960,7 +1477,7 @@ module Merb end end end - + deps end @@ -968,12 +1485,13 @@ module Merb if File.exists?(config_file) self.class.parse_dependencies_yaml(File.read(config_file)) else + warning "No dependencies.yml file found at: #{config_file}" [] end end - def use_config? - options[:config] || options[:"config-file"] + def extract_dependencies? + options[:extract] || extract_dependencies end def config_file @@ -1025,20 +1543,18 @@ module Merb end def edge_strategy(deps) + use_edge_gem_server installed_from_rubygems = [] # Selectively update repositories for the matching dependencies update_dependency_repositories(deps) unless dry_run? - # Skip gem dependencies to prevent them from being installed from stable; - # however, core dependencies will be retrieved from source when available - install_opts = { :ignore_dependencies => true } if core = deps.find { |d| d.name == 'merb-core' } if dry_run? note "Installing #{core.name}..." else - if install_dependency_from_source(core, install_opts) - elsif install_dependency(core, install_opts) + if install_dependency_from_source(core) + elsif install_dependency(core) info "Installed #{core.name} from rubygems..." installed_from_rubygems << core end @@ -1050,8 +1566,8 @@ module Merb if dry_run? note "Installing #{dependency.name}..." else - if install_dependency_from_source(dependency, install_opts) - elsif install_dependency(dependency, install_opts) + if install_dependency_from_source(dependency) + elsif install_dependency(dependency) info "Installed #{dependency.name} from rubygems..." installed_from_rubygems << dependency end @@ -1072,26 +1588,29 @@ module Merb end # Extract application dependencies by querying the app directly. - def self.extract_dependencies(merb_root, env = 'production') + def self.extract_dependencies(merb_root) require 'merb-core' if !@_merb_loaded || Merb.root != merb_root Merb.start_environment( + :log_level => :fatal, :testing => true, :adapter => 'runner', - :environment => env, + :environment => ENV['MERB_ENV'] || 'development', :merb_root => merb_root ) @_merb_loaded = true end Merb::BootLoader::Dependencies.dependencies - rescue => e + rescue StandardError => e error "Couldn't extract dependencies from application!" error e.message - - p e.backtrace - - puts "Make sure you're executing the task from your app (--merb-root), or" - puts "specify a config option (--config or --config-file=YAML_FILE)" + puts "Make sure you're executing the task from your app (--merb-root)" + return [] + rescue SystemExit + error "Couldn't extract dependencies from application!" + error "application failed to run" + puts "Please check if your application runs using 'merb'; for example," + puts "look for any gem version mismatches in dependencies.rb" return [] end @@ -1111,25 +1630,29 @@ module Merb dependencies end - end + end class Stack < Thor + group 'core' + # The Stack tasks will install dependencies based on known sets of gems, # regardless of actual application dependency settings. DM_STACK = %w[ + extlib + data_objects dm-core dm-aggregates dm-migrations dm-timestamps dm-types dm-validations + merb_datamapper ] MERB_STACK = %w[ - minigems - + extlib merb-core merb-action-args merb-assets @@ -1138,11 +1661,15 @@ module Merb merb-mailer merb-slices merb-auth + merb-auth-core + merb-auth-more + merb-auth-slice-password + merb-param-protection + merb-exceptions ] + DM_STACK MERB_BASICS = %w[ - minigems - + extlib merb-core merb-action-args merb-assets @@ -1215,13 +1742,22 @@ module Merb dm-rest-adapter ] + DATA_OBJECTS = %w[ + data_objects + do_derby do_hsqldb + do_jdbc + do_mysql + do_postgres + do_sqlite3 + ] + attr_accessor :system, :local, :missing include MerbThorHelper global_method_options = { "--merb-root" => :optional, # the directory to operate on - "--include-dependencies" => :boolean, # gather sub-dependencies + "--ignore-dependencies" => :boolean, # skip sub-dependencies "--version" => :optional # gather specific version of framework } @@ -1270,6 +1806,7 @@ module Merb "--dry-run" => :boolean, "--strategy" => :optional def install(*comps) + use_edge_gem_server if options[:edge] mngr = self.dependency_manager deps = gather_dependencies(comps) mngr.system, mngr.local, mngr.missing = Merb::Gem.partition_dependencies(deps, gem_dir) @@ -1281,6 +1818,7 @@ module Merb # See also: Merb::Dependencies#uninstall # # Examples: + # # merb:stack:uninstall # uninstall the default merb stack # merb:stack:uninstall merb-more # uninstall merb-more # merb:stack:uninstall merb-core thor merb-slices # uninstall the specified gems @@ -1294,6 +1832,48 @@ module Merb clobber_dependencies! end + # Install or uninstall minigems from the system. + # + # Due to the specific nature of MiniGems it can only be installed system-wide. + # + # Examples: + # + # merb:stack:minigems install # install minigems + # merb:stack:minigems uninstall # uninstall minigems + + desc 'minigems (install|uninstall)', 'Install or uninstall minigems (needs sudo privileges)' + def minigems(action) + case action + when 'install' + Kernel.system "#{sudo} thor merb:stack:install_minigems" + when 'uninstall' + Kernel.system "#{sudo} thor merb:stack:uninstall_minigems" + else + error "Invalid command: merb:stack:minigems #{action}" + end + end + + # hidden minigems install task + def install_minigems + message "Installing MiniGems" + mngr = self.dependency_manager + deps = gather_dependencies('minigems') + mngr.system, mngr.local, mngr.missing = Merb::Gem.partition_dependencies(deps, gem_dir) + mngr.force_gem_dir = ::Gem.dir + mngr.install_dependencies(strategy, deps) + Kernel.system "#{sudo} minigem install" + end + + # hidden minigems uninstall task + def uninstall_minigems + message "Uninstalling MiniGems" + Kernel.system "#{sudo} minigem uninstall" + deps = gather_dependencies('minigems') + self.system, self.local, self.missing = Merb::Gem.partition_dependencies(deps, gem_dir) + # Clobber existing local dependencies - based on self.local + clobber_dependencies! + end + protected def gather_dependencies(comps = []) @@ -1342,6 +1922,7 @@ module Merb comps["merb-more"] = MERB_MORE.sort comps["merb-plugins"] = MERB_PLUGINS.sort comps["dm-more"] = DM_MORE.sort + comps["do"] = DATA_OBJECTS.sort comps end @@ -1367,7 +1948,7 @@ module Merb end def self.framework_components - %w[merb-core merb-more merb-plugins].inject([]) do |all, comp| + %w[merb-core merb-more].inject([]) do |all, comp| all + components(comp) end end @@ -1389,7 +1970,7 @@ module Merb end def self.base_components - %w[thor rake] + %w[thor rake extlib] end def self.all_components @@ -1401,6 +1982,7 @@ module Merb def self.core_dependencies(gem_dir = nil, ignore_deps = false) @_core_dependencies ||= begin if gem_dir # add local gems to index + orig_gem_path = ::Gem.path ::Gem.clear_paths; ::Gem.path.unshift(gem_dir) end deps = [] @@ -1415,7 +1997,7 @@ module Merb deps += gemspec.dependencies end end - ::Gem.clear_paths if gem_dir # reset + ::Gem.path.replace(orig_gem_path) if gem_dir # reset deps end end @@ -1435,476 +2017,4 @@ module Merb end - class Tasks < Thor - - include MerbThorHelper - - # Show merb.thor version information - # - # merb:tasks:version # show the current version info - # merb:tasks:version --info # show extended version info - - desc 'version', 'Show verion info' - method_options "--info" => :boolean - def version - message "Currently installed merb.thor version: #{MERB_THOR_VERSION}" - if options[:version] - self.options = { :"dry-run" => true } - self.update # run update task with dry-run enabled - end - end - - # Update merb.thor tasks from remotely available version - # - # merb:tasks:update # update merb.thor - # merb:tasks:update --force # force-update merb.thor - # merb:tasks:update --dry-run # show version info only - - desc 'update [URL]', 'Fetch the latest merb.thor and install it locally' - method_options "--dry-run" => :boolean, "--force" => :boolean - def update(url = 'http://merbivore.com/merb.thor') - require 'open-uri' - require 'rubygems/version' - remote_file = open(url) - code = remote_file.read - - # Extract version information from the source code - if version = code[/^MERB_THOR_VERSION\s?=\s?('|")([\.\d]+)('|")/,2] - # borrow version comparison from rubygems' Version class - current_version = ::Gem::Version.new(MERB_THOR_VERSION) - remote_version = ::Gem::Version.new(version) - - if current_version >= remote_version - puts "currently installed: #{current_version}" - if current_version != remote_version - puts "available version: #{remote_version}" - end - info "No update of merb.thor necessary#{options[:force] ? ' (forced)' : ''}" - proceed = options[:force] - elsif current_version < remote_version - puts "currently installed: #{current_version}" - puts "available version: #{remote_version}" - proceed = true - end - - if proceed && !dry_run? - File.open(File.join(__FILE__), 'w') do |f| - f.write(code) - end - success "Installed the latest merb.thor (v#{version})" - end - else - raise "invalid source-code data" - end - rescue OpenURI::HTTPError - error "Error opening #{url}" - rescue => e - error "An error occurred (#{e.message})" - end - - end - - #### MORE LOW-LEVEL TASKS #### - - class Gem < Thor - - group 'core' - - include MerbThorHelper - extend GemManagement - - attr_accessor :system, :local, :missing - - global_method_options = { - "--merb-root" => :optional, # the directory to operate on - "--version" => :optional, # gather specific version of gem - "--ignore-dependencies" => :boolean # don't install sub-dependencies - } - - method_options global_method_options - def initialize(*args); super; end - - # List gems that match the specified criteria. - # - # By default all local gems are listed. When the first argument is 'all' the - # list is partitioned into system an local gems; specify 'system' to show - # only system gems. A second argument can be used to filter on a set of known - # components, like all merb-more gems for example. - # - # Examples: - # - # merb:gem:list # list all local gems - the default - # merb:gem:list all # list system and local gems - # merb:gem:list system # list only system gems - # merb:gem:list all merb-more # list only merb-more related gems - # merb:gem:list --version 0.9.8 # list gems that match the version - - desc 'list [all|local|system] [comp]', 'Show installed gems' - def list(filter = 'local', comp = nil) - deps = comp ? Merb::Stack.select_component_dependencies(dependencies, comp) : dependencies - self.system, self.local, self.missing = Merb::Gem.partition_dependencies(deps, gem_dir) - case filter - when 'all' - message 'Installed system gems:' - display_gemspecs(system) - message 'Installed local gems:' - display_gemspecs(local) - when 'system' - message 'Installed system gems:' - display_gemspecs(system) - when 'local' - message 'Installed local gems:' - display_gemspecs(local) - else - warning "Invalid listing filter '#{filter}'" - end - end - - # Install the specified gems. - # - # All arguments should be names of gems to install. - # - # When :force => true then any existing versions of the gems to be installed - # will be uninstalled first. It's important to note that so-called meta-gems - # or gems that exactly match a set of Merb::Stack.components will have their - # sub-gems uninstalled too. For example, uninstalling merb-more will install - # all contained gems: merb-action-args, merb-assets, merb-gen, ... - # - # Examples: - # - # merb:gem:install merb-core merb-slices # install all specified gems - # merb:gem:install merb-core --version 0.9.8 # install a specific version of a gem - # merb:gem:install merb-core --force # uninstall then subsequently install the gem - # merb:gem:install merb-core --cache # try to install locally from system gems - # merb:gem:install merb-core --binaries # also install adapted bin wrapper - - desc 'install GEM_NAME [GEM_NAME, ...]', 'Install a gem from rubygems' - method_options "--cache" => :boolean, - "--binaries" => :boolean, - "--dry-run" => :boolean, - "--force" => :boolean - def install(*names) - self.include_dependencies = true # deal with dependencies by default - opts = { :version => options[:version], :cache => options[:cache] } - current_gem = nil - - # uninstall existing gems of the ones we're going to install - uninstall(*names) if options[:force] - - names.each do |gem_name| - current_gem = gem_name - if dry_run? - note "Installing #{current_gem}..." - else - message "Installing #{current_gem}..." - self.class.install(gem_name, default_install_options.merge(opts)) - ensure_bin_wrapper_for(gem_name) if options[:binaries] - end - end - rescue => e - error "Failed to install #{current_gem ? current_gem : 'gem'} (#{e.message})" - end - - # Uninstall the specified gems. - # - # By default all specified gems are uninstalled. It's important to note that - # so-called meta-gems or gems that match a set of Merb::Stack.components will - # have their sub-gems uninstalled too. For example, uninstalling merb-more - # will install all contained gems: merb-action-args, merb-assets, ... - # - # Existing dependencies will be clobbered; when :force => true then all gems - # will be cleared, otherwise only existing local dependencies of the - # matching component set will be removed. - # - # Examples: - # - # merb:gem:uninstall merb-core merb-slices # uninstall all specified gems - # merb:gem:uninstall merb-core --version 0.9.8 # uninstall a specific version of a gem - - desc 'uninstall GEM_NAME [GEM_NAME, ...]', 'Unstall a gem' - method_options "--dry-run" => :boolean - def uninstall(*names) - self.include_dependencies = true # deal with dependencies by default - opts = { :version => options[:version] } - current_gem = nil - if dry_run? - note "Uninstalling any existing gems of: #{names.join(', ')}" - else - message "Uninstalling any existing gems of: #{names.join(', ')}" - names.each do |gem_name| - current_gem = gem_name - Merb::Gem.uninstall(gem_name, default_uninstall_options) rescue nil - # if this gem is a meta-gem or a component set name, remove sub-gems - (Merb::Stack.components(gem_name) || []).each do |comp| - Merb::Gem.uninstall(comp, default_uninstall_options) rescue nil - end - end - end - rescue => e - error "Failed to uninstall #{current_gem ? current_gem : 'gem'} (#{e.message})" - end - - private - - # Return dependencies for all installed gems; both system-wide and locally; - # optionally filters on :version requirement. - def dependencies - version_req = if options[:version] - ::Gem::Requirement.create(options[:version]) - else - ::Gem::Requirement.default - end - if gem_dir - ::Gem.clear_paths; ::Gem.path.unshift(gem_dir) - ::Gem.source_index.refresh! - end - deps = [] - ::Gem.source_index.each do |fullname, gemspec| - if version_req.satisfied_by?(gemspec.version) - deps << ::Gem::Dependency.new(gemspec.name, "= #{gemspec.version}") - end - end - ::Gem.clear_paths if gem_dir - deps.sort - end - - public - - # Install gem with some default options. - def self.install(name, options = {}) - defaults = {} - defaults[:cache] = false unless opts[:install_dir] - install_gem(name, defaults.merge(options)) - end - - # Uninstall gem with some default options. - def self.uninstall(name, options = {}) - defaults = { :ignore => true, :executables => true } - uninstall_gem(name, defaults.merge(options)) - end - - end - - class Source < Thor - - group 'core' - - include MerbThorHelper - extend GemManagement - - attr_accessor :system, :local, :missing - - global_method_options = { - "--merb-root" => :optional, # the directory to operate on - "--include-dependencies" => :boolean, # gather sub-dependencies - "--sources" => :optional # a yml config to grab sources from - } - - method_options global_method_options - def initialize(*args); super; end - - # List source repositories, of either local or known sources. - # - # Examples: - # - # merb:source:list # list all local sources - # merb:source:list available # list all known sources - - desc 'list [local|available]', 'Show git source repositories' - def list(mode = 'local') - if mode == 'available' - message 'Available source repositories:' - repos = self.class.repos(options[:sources]) - repos.keys.sort.each { |name| puts "- #{name}: #{repos[name]}" } - elsif mode == 'local' - message 'Current source repositories:' - Dir[File.join(source_dir, '*')].each do |src| - next unless File.directory?(src) - src_name = File.basename(src) - unless (repos = source_manager.existing_repos(src_name)).empty? - puts "#{src_name}" - repos.keys.sort.each { |b| puts "- #{b}: #{repos[b]}" } - end - end - else - error "Unknown listing: #{mode}" - end - end - - # Install the specified gems. - # - # All arguments should be names of gems to install. - # - # When :force => true then any existing versions of the gems to be installed - # will be uninstalled first. It's important to note that so-called meta-gems - # or gems that exactly match a set of Merb::Stack.components will have their - # sub-gems uninstalled too. For example, uninstalling merb-more will install - # all contained gems: merb-action-args, merb-assets, merb-gen, ... - # - # Examples: - # - # merb:source:install merb-core merb-slices # install all specified gems - # merb:source:install merb-core --force # uninstall then subsequently install the gem - # merb:source:install merb-core --wipe # clear repo then install the gem - # merb:source:install merb-core --binaries # also install adapted bin wrapper - - desc 'install GEM_NAME [GEM_NAME, ...]', 'Install a gem from git source/edge' - method_options "--binaries" => :boolean, - "--dry-run" => :boolean, - "--force" => :boolean, - "--wipe" => :boolean - def install(*names) - # uninstall existing gems of the ones we're going to install - uninstall(*names) if options[:force] || options[:wipe] - - # We want dependencies instead of just names - deps = names.map { |n| ::Gem::Dependency.new(n, ::Gem::Requirement.default) } - - # Selectively update repositories for the matching dependencies - update_dependency_repositories(deps) unless dry_run? - - current_gem = nil - deps.each do |dependency| - current_gem = dependency.name - if dry_run? - note "Installing #{current_gem} from source..." - else - message "Installing #{current_gem} from source..." - if install_dependency_from_source(dependency) - ensure_bin_wrapper_for(dependency.name) if options[:binaries] - end - end - end - rescue => e - error "Failed to install #{current_gem ? current_gem : 'gem'} (#{e.message})" - end - - # Uninstall the specified gems. - # - # By default all specified gems are uninstalled. It's important to note that - # so-called meta-gems or gems that match a set of Merb::Stack.components will - # have their sub-gems uninstalled too. For example, uninstalling merb-more - # will install all contained gems: merb-action-args, merb-assets, ... - # - # Existing dependencies will be clobbered; when :force => true then all gems - # will be cleared, otherwise only existing local dependencies of the - # matching component set will be removed. Additionally when :wipe => true, - # the matching git repositories will be removed from the source directory. - # - # Examples: - # - # merb:source:uninstall merb-core merb-slices # uninstall all specified gems - # merb:source:uninstall merb-core --wipe # force-uninstall a gem and clear repo - - desc 'uninstall GEM_NAME [GEM_NAME, ...]', 'Unstall a gem (specify --force to remove the repo)' - method_options "--version" => :optional, "--dry-run" => :boolean, "--wipe" => :boolean - def uninstall(*names) - # Remove the repos that contain the gem - if options[:wipe] - extract_repositories(names).each do |(name, url)| - if File.directory?(src = File.join(source_dir, name)) - if dry_run? - note "Removing #{src}..." - else - info "Removing #{src}..." - FileUtils.rm_rf(src) - end - end - end - end - - # Use the Merb::Gem#uninstall task to handle this - gem_tasks = Merb::Gem.new - gem_tasks.options = options - gem_tasks.uninstall(*names) - end - - # Update the specified source repositories. - # - # The arguments can be actual repository names (from Merb::Source.repos) - # or names of known merb stack gems. If the repo doesn't exist already, - # it will be created and cloned. - # - # merb:source:pull merb-core # update source of specified gem - # merb:source:pull merb-slices # implicitly updates merb-more - - desc 'pull REPO_NAME [GEM_NAME, ...]', 'Update git source repository from edge' - def pull(*names) - repos = extract_repositories(names) - update_repositories(repos) - unless repos.empty? - message "Updated the following repositories:" - repos.each { |name, url| puts "- #{name}: #{url}" } - else - warning "No repositories found to update!" - end - end - - # Clone a git repository into ./src. - - # The repository can be a direct git url or a known -named- repository. - # - # Examples: - # - # merb:source:clone merb-core - # merb:source:clone dm-core awesome-repo - # merb:source:clone dm-core --sources ./path/to/sources.yml - # merb:source:clone git://github.com/sam/dm-core.git - - desc 'clone (REPO_NAME|URL) [DIR_NAME]', 'Clone git source repository by name or url' - def clone(repository, name = nil) - if repository =~ /^git:\/\// - repository_url = repository - repository_name = File.basename(repository_url, '.git') - elsif url = Merb::Source.repo(repository, options[:sources]) - repository_url = url - repository_name = repository - end - source_manager.clone(name || repository_name, repository_url) - end - - # Git repository sources - pass source_config option to load a yaml - # configuration file - defaults to ./config/git-sources.yml and - # ~/.merb/git-sources.yml - which you need to create yourself. - # - # Example of contents: - # - # merb-core: git://github.com/myfork/merb-core.git - # merb-more: git://github.com/myfork/merb-more.git - - def self.repos(source_config = nil) - source_config ||= begin - local_config = File.join(Dir.pwd, 'config', 'git-sources.yml') - user_config = File.join(ENV["HOME"] || ENV["APPDATA"], '.merb', 'git-sources.yml') - File.exists?(local_config) ? local_config : user_config - end - if source_config && File.exists?(source_config) - default_repos.merge(YAML.load(File.read(source_config))) - else - default_repos - end - end - - def self.repo(name, source_config = nil) - self.repos(source_config)[name] - end - - # Default Git repositories - def self.default_repos - @_default_repos ||= { - 'merb' => "git://github.com/wycats/merb.git", - 'merb-plugins' => "git://github.com/wycats/merb-plugins.git", - 'extlib' => "git://github.com/sam/extlib.git", - 'dm-core' => "git://github.com/sam/dm-core.git", - 'dm-more' => "git://github.com/sam/dm-more.git", - 'sequel' => "git://github.com/wayneeseguin/sequel.git", - 'do' => "git://github.com/sam/do.git", - 'thor' => "git://github.com/wycats/thor.git", - 'rake' => "git://github.com/jimweirich/rake.git", - 'minigems' => "git://github.com/fabien/minigems.git" - } - end - - end - end \ No newline at end of file tasks/merb.thor 22010ad2502aaa1f54d54a0c174b27f00a02a547 182ae72c825098bf22b9e539930280cb25314860 John Nunemaker nunemaker@gmail.com http://github.com/jnunemaker/astrotrain/commit/eb28bce1a6d4e755d58cb4dacb4413871c865b3a eb28bce1a6d4e755d58cb4dacb4413871c865b3a 2009-04-19T19:56:58-07:00 2009-04-19T19:56:58-07:00 Merge branch 'integration' of git@github.com:jnunemaker/astrotrain into integration Conflicts: app/models/logged_mail.rb app/models/mapping.rb app/models/mapping/http_post.rb app/models/mapping/jabber.rb app/models/message.rb config/dependencies.rb config/init.rb spec/models/logged_mail_spec.rb spec/models/message_spec.rb spec/models/transport_spec.rb 5f25bd65f674fb1041f88b85280ea2848c3336ba John Nunemaker nunemaker@gmail.com